Skip to content

Commit 5847f5b

Browse files
authored
Reduce complexity of CriticalPathQueuingDurationDataProvider (#194)
This provider currently performs several full loops over the Bazel profiles. This results in it not scaling well performance and memory consumption-wise. Also at the profile level, some expensive copying and re-sorting of bazel profile data is performed each time it is requested by the providers. Allocations: <img width="3079" height="828" alt="image" src="https://github.com/user-attachments/assets/164a37a3-8193-42fb-a860-6bc961d277ac" /> CPU: <img width="3098" height="988" alt="image" src="https://github.com/user-attachments/assets/13ed0e2c-f0b4-42d4-8562-04c000f76325" /> This PR aims to reduce the amount of loops over the data in `CriticalPathQueuingDurationDataProvider` to just one, enabling it to scale better. Also, cleans the expensive copying and re-sorting of bazel profile data each time it is requested, by switching thread building to a Builder pattern, which sorts the entries internally as needed before returning the immutable sorted result. Thanks to @iamricard for doing a lot of the initial work and investigation on this (the branch was opened in another repository so I didn't know how to cleanly credit him in the commits I moved over to here) --------- Signed-off-by: Felipe <afrueda97@outlook.com>
1 parent 3fa9602 commit 5847f5b

6 files changed

Lines changed: 500 additions & 328 deletions

File tree

analyzer/java/com/engflow/bazel/invocation/analyzer/bazelprofile/BazelProfile.java

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,14 @@
3535
import java.io.IOException;
3636
import java.io.InputStream;
3737
import java.io.InputStreamReader;
38+
import java.io.Reader;
3839
import java.nio.charset.StandardCharsets;
3940
import java.util.HashMap;
4041
import java.util.List;
4142
import java.util.Map;
4243
import java.util.Optional;
4344
import java.util.concurrent.atomic.AtomicLong;
45+
import java.util.stream.Collectors;
4446
import java.util.stream.Stream;
4547
import java.util.zip.GZIPInputStream;
4648
import java.util.zip.ZipException;
@@ -82,12 +84,17 @@ public static BazelProfile createFromInputStream(InputStream inputStream)
8284
new JsonReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)));
8385
}
8486

87+
public static BazelProfile of(Reader reader) {
88+
return new BazelProfile(new JsonReader(reader));
89+
}
90+
8591
private final BazelVersion bazelVersion;
8692
private final Map<String, String> otherData = new HashMap<>();
87-
private final Map<ThreadId, ProfileThread> threads = new HashMap<>();
93+
private final Map<ThreadId, ProfileThread> threads;
8894

8995
private BazelProfile(JsonReader profileReader) {
9096
try {
97+
Map<ThreadId, ProfileThread.Builder> threadBuilders = new HashMap<>();
9198
boolean hasOtherData = false;
9299
boolean hasTraceEvents = false;
93100
profileReader.beginObject();
@@ -116,17 +123,17 @@ private BazelProfile(JsonReader profileReader) {
116123
continue;
117124
}
118125
ThreadId threadId = new ThreadId(pid, tid);
119-
ProfileThread profileThread =
120-
threads.compute(
126+
ProfileThread.Builder profileThreadBuilder =
127+
threadBuilders.compute(
121128
threadId,
122129
(key, t) -> {
123130
if (t == null) {
124-
t = new ProfileThread(threadId);
131+
t = new ProfileThread.Builder().setThreadId(key);
125132
}
126133
return t;
127134
});
128135
// TODO: Use success response to take action on errant events.
129-
profileThread.addEvent(traceEvent);
136+
profileThreadBuilder.addEvent(traceEvent);
130137
}
131138
profileReader.endArray();
132139
break;
@@ -143,6 +150,9 @@ private BazelProfile(JsonReader profileReader) {
143150
TraceEventFormatConstants.SECTION_OTHER_DATA,
144151
TraceEventFormatConstants.SECTION_TRACE_EVENTS));
145152
}
153+
threads =
154+
threadBuilders.entrySet().stream()
155+
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build()));
146156
} catch (IllegalStateException | IOException e) {
147157
throw new IllegalArgumentException("Could not parse Bazel profile.", e);
148158
}

0 commit comments

Comments
 (0)