Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 44 additions & 8 deletions Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,9 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis
/// Runs the algorithm speed analysis against the speed metrics tracked for the backtest,
/// falling back to the completion log line when they cannot measure the speed.
/// </summary>
public override IReadOnlyList<QuantConnect.Analysis> Run(ResultsAnalysisRunParameters parameters) => Run(parameters.Speed, parameters.Logs);
public override IReadOnlyList<QuantConnect.Analysis> Run(ResultsAnalysisRunParameters parameters)
=> Run(parameters.Speed, parameters.Logs, parameters.Language,
performanceTrackingEnabled: parameters.Algorithm?.Settings.PerformanceSamplePeriod > TimeSpan.Zero);

/// <summary>
/// Runs the algorithm speed analysis against the given speed metrics.
Expand All @@ -138,22 +140,26 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis
/// </summary>
/// <param name="speed">The speed metrics tracked for the running backtest, or null when not tracked.</param>
/// <param name="logs">The log lines to search for the completion line, or null when not available.</param>
/// <param name="language">The programming language the algorithm is written in.</param>
/// <param name="performanceTrackingEnabled">Whether the algorithm already has performance tracking enabled,
/// so the findings don't suggest enabling it again.</param>
/// <returns>The failed sub-findings, or empty when no speed condition failed or none could be measured.</returns>
public IReadOnlyList<QuantConnect.Analysis> Run(AlgorithmSpeedTracker speed, IReadOnlyList<string> logs = null)
public IReadOnlyList<QuantConnect.Analysis> Run(AlgorithmSpeedTracker speed, IReadOnlyList<string> logs = null,
Language language = Language.CSharp, bool performanceTrackingEnabled = false)
{
var findings = new List<QuantConnect.Analysis>();
var speedMeasured = false;
if (speed != null && speed.SampledSpan >= MinimumSampledSpan)
{
speedMeasured = AddSlowExecution(speed, findings);
speedMeasured = AddSlowExecution(speed, findings, language, performanceTrackingEnabled);
AddLongProjectedRuntime(speed, findings);
AddThroughputDegradation(speed, findings);
AddThroughputDegradation(speed, findings, language, performanceTrackingEnabled);
AddHistoryRequestLoad(speed, findings);
}

if (!speedMeasured)
{
AddSlowExecutionFromCompletionLog(logs, findings);
AddSlowExecutionFromCompletionLog(logs, findings, language, performanceTrackingEnabled);
}

return CreateAggregatedResponse(findings);
Expand All @@ -163,7 +169,8 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis
/// Reports slow execution when the recent data points per second are below the platform benchmark.
/// </summary>
/// <returns>Whether the speed could be measured, regardless of it being slow or not.</returns>
private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List<QuantConnect.Analysis> findings)
private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List<QuantConnect.Analysis> findings,
Language language, bool performanceTrackingEnabled)
{
if (!speed.HasDataPointCounts)
{
Expand Down Expand Up @@ -208,6 +215,8 @@ private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List<QuantConn
[
"Review the algorithm code for inefficiencies.",

.. PerformanceTrackingSolutions(language, performanceTrackingEnabled),

"If there is a universe, reduce its size.",

"Reduce the data resolution.",
Expand All @@ -225,7 +234,8 @@ private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List<QuantConn
/// logged once the backtest ends, so in-run log deltas never match and the fallback can
/// only fire on the final analysis.
/// </summary>
private static void AddSlowExecutionFromCompletionLog(IReadOnlyList<string> logs, List<QuantConnect.Analysis> findings)
private static void AddSlowExecutionFromCompletionLog(IReadOnlyList<string> logs, List<QuantConnect.Analysis> findings,
Language language, bool performanceTrackingEnabled)
{
for (var i = (logs?.Count ?? 0) - 1; i >= 0; i--)
{
Expand All @@ -247,6 +257,8 @@ private static void AddSlowExecutionFromCompletionLog(IReadOnlyList<string> logs
[
"Review the algorithm code for inefficiencies.",

.. PerformanceTrackingSolutions(language, performanceTrackingEnabled),

"If there is a universe, reduce its size.",

"Reduce the data resolution.",
Expand Down Expand Up @@ -315,7 +327,8 @@ private static void AddLongProjectedRuntime(AlgorithmSpeedTracker speed, List<Qu
/// <see cref="DegradationRatio"/> of the early-run baseline. Requires enough samples for the
/// baseline and recent windows to not overlap.
/// </summary>
private static void AddThroughputDegradation(AlgorithmSpeedTracker speed, List<QuantConnect.Analysis> findings)
private static void AddThroughputDegradation(AlgorithmSpeedTracker speed, List<QuantConnect.Analysis> findings,
Language language, bool performanceTrackingEnabled)
{
if (!speed.HasDataPointCounts || speed.SampleCount < 2 * AlgorithmSpeedTracker.RecentWindowSamples + 1)
{
Expand Down Expand Up @@ -344,6 +357,8 @@ private static void AddThroughputDegradation(AlgorithmSpeedTracker speed, List<Q
"If there is a universe, check whether the number of selected securities keeps growing; remove securities that are no longer used.",

"Check the algorithm's memory usage: sustained growth causes garbage collection pressure that slows the whole run down.",

.. PerformanceTrackingSolutions(language, performanceTrackingEnabled),
]));
}

Expand Down Expand Up @@ -374,6 +389,27 @@ private static void AddHistoryRequestLoad(AlgorithmSpeedTracker speed, List<Quan
]));
}

/// <summary>
/// The performance tracking suggestion shared by the findings whose diagnosis needs to locate
/// where the execution time is spent: setting <see cref="AlgorithmSettings.PerformanceSamplePeriod"/>
/// adds a "Performance" chart with the engine's time breakdown on the next run.
/// Empty when the algorithm already has performance tracking enabled.
/// </summary>
private static IEnumerable<string> PerformanceTrackingSolutions(Language language, bool performanceTrackingEnabled)
{
if (performanceTrackingEnabled)
{
yield break;
}

yield return $"To see where the execution time is spent, set the " +
$"`{FormatCode(nameof(AlgorithmSettings.PerformanceSamplePeriod), language)}` setting, like " +
(language == Language.Python
? "`self.settings.performance_sample_period = timedelta(days=1)`"
: "`Settings.PerformanceSamplePeriod = TimeSpan.FromDays(1);`") +
", and rerun to get a \"Performance\" time-breakdown chart.";
}

/// <summary>
/// Formats a data points per second rate compactly: in thousands like "12.5k" when at least
/// one thousand, as a raw count like "340" below that, so very slow rates don't read as "0.0k".
Expand Down
5 changes: 4 additions & 1 deletion Engine/Results/BacktestingResultHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,10 @@ private void Update()
runtimeStatistics,
new Dictionary<string, AlgorithmPerformance>(),
// we store the last 100 order events, the final packet will contain the full list
TransactionHandler.OrderEvents.Reverse().Take(100).ToList(), state: GetAlgorithmState()));
TransactionHandler.OrderEvents.Reverse().Take(100).ToList(), state: GetAlgorithmState()))
{
ServerStatistics = serverStatistics
};

if (RunResultsAnalysis)
{
Expand Down
56 changes: 56 additions & 0 deletions Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,62 @@ public void FlagsHistoryRequestDominatedLoad()
StringAssert.Contains("75% of the data points", (string)finding.Sample);
}

[Test]
public void SlowFindingsSuggestEnablingPerformanceTracking()
{
// Both slow and degrading: the slow execution and the degradation findings carry the suggestion
var tracker = new AlgorithmSpeedTracker();
var dataPoints = 0L;
for (var i = 0; i < 12; i++)
{
tracker.AddSample(new(TimeSpan.FromSeconds(30 * i), dataPoints, 0, 0, 0));
dataPoints += i < 6 ? 3_000_000 : 300_000;
}

var findings = new AlgorithmSpeedAnalysis().Run(tracker);

Assert.AreEqual(2, findings.Count);
foreach (var finding in findings)
{
Assert.IsTrue(finding.Solutions.Any(solution =>
solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal)));
}

// The completion log fallback carries it too
var fallbackFinding = new AlgorithmSpeedAnalysis().Run(null, new[] { SlowCompletionLogLine }).Single();
Assert.IsTrue(fallbackFinding.Solutions.Any(solution =>
solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal)));
}

[Test]
public void PerformanceTrackingSuggestionFollowsTheAlgorithmLanguage()
{
var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30,
dataPointsPerStep: 300_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10);

var finding = new AlgorithmSpeedAnalysis().Run(tracker, language: Language.Python).Single();

var solution = finding.Solutions.Single(solution => solution.Contains("performance_sample_period", StringComparison.Ordinal));
StringAssert.Contains("self.settings.performance_sample_period", solution);
}

[Test]
public void NoPerformanceTrackingSuggestionWhenAlreadyEnabled()
{
var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30,
dataPointsPerStep: 300_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10);

var finding = new AlgorithmSpeedAnalysis().Run(tracker, performanceTrackingEnabled: true).Single();

Assert.IsFalse(finding.Solutions.Any(solution =>
solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal)));

var fallbackFinding = new AlgorithmSpeedAnalysis()
.Run(null, new[] { SlowCompletionLogLine }, performanceTrackingEnabled: true).Single();
Assert.IsFalse(fallbackFinding.Solutions.Any(solution =>
solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal)));
}

[Test]
public void NoHistoryLoadFindingBelowTheMinimumHistoryDataPointCount()
{
Expand Down
Loading