Skip to content

Commit dfa101e

Browse files
Flossyclaude
andcommitted
fix: add size validation and timeout configuration to HdfsClassSource
- Add MAX_CLASS_SIZE constant (10MB default, configurable) - Check file size before downloading to prevent OOM - Add timeout configuration (30s socket, 10s connect) - Configure HDFS client timeouts to prevent infinite hangs - Add maxClassSize(), socketTimeout(), connectTimeout() builder methods - Use correct property name (dfs.client.socket-timeout, not deprecated dfs.socket.timeout) Without these fixes: - Multi-GB files could cause OutOfMemoryError - Network issues could hang threads forever - No recovery without kill -9 Fixes #52 Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent e01560f commit dfa101e

1 file changed

Lines changed: 73 additions & 9 deletions

File tree

src/main/java/org/flossware/classloader/filesystem/HdfsClassSource.java

Lines changed: 73 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import org.apache.hadoop.conf.Configuration;
44

55
import static org.flossware.classloader.util.ClassLoaderConstants.DEFAULT_BUFFER_SIZE;
6+
import org.apache.hadoop.fs.FileStatus;
67
import org.apache.hadoop.fs.FileSystem;
78
import org.apache.hadoop.fs.Path;
89
import org.flossware.classloader.ClassSource;
@@ -19,28 +20,58 @@
1920
* Requires the Hadoop client dependency.
2021
*/
2122
public class HdfsClassSource implements ClassSource, AutoCloseable {
23+
private static final long MAX_CLASS_SIZE = 10 * 1024 * 1024; // 10MB default
24+
private static final int DEFAULT_SOCKET_TIMEOUT = 30000; // 30 seconds
25+
private static final int DEFAULT_CONNECT_TIMEOUT = 10000; // 10 seconds
26+
2227
private final FileSystem hdfs;
2328
private final String basePath;
29+
private final long maxClassSize;
2430

25-
private HdfsClassSource(FileSystem hdfs, String basePath) {
31+
private HdfsClassSource(FileSystem hdfs, String basePath, long maxClassSize) {
2632
this.hdfs = Objects.requireNonNull(hdfs, "hdfs cannot be null");
2733
this.basePath = basePath != null ? basePath : "/";
34+
this.maxClassSize = maxClassSize;
2835
}
2936

3037
@Override
3138
public byte[] loadClassData(String className) throws IOException {
3239
Path classPath = getClassPath(className);
3340

34-
try (InputStream in = hdfs.open(classPath);
35-
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
41+
// Check size BEFORE downloading to prevent OOM
42+
FileStatus status = hdfs.getFileStatus(classPath);
43+
long size = status.getLen();
44+
45+
if (size > maxClassSize) {
46+
throw new IOException(
47+
"Class file too large: " + size + " bytes (max " + maxClassSize + ")"
48+
);
49+
}
50+
51+
if (size > Integer.MAX_VALUE) {
52+
throw new IOException(
53+
"Class file exceeds Java array limit: " + size + " bytes"
54+
);
55+
}
56+
57+
// Safe to download - size is within limits
58+
try (InputStream in = hdfs.open(classPath)) {
59+
byte[] data = new byte[(int)size];
60+
int totalRead = 0;
3661

37-
byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
38-
int bytesRead;
39-
while ((bytesRead = in.read(buffer)) != -1) {
40-
out.write(buffer, 0, bytesRead);
62+
while (totalRead < size) {
63+
int n = in.read(data, totalRead, (int)size - totalRead);
64+
if (n == -1) break;
65+
totalRead += n;
4166
}
4267

43-
return out.toByteArray();
68+
if (totalRead != size) {
69+
throw new IOException(
70+
"Expected " + size + " bytes but read " + totalRead
71+
);
72+
}
73+
74+
return data;
4475
}
4576
}
4677

@@ -82,6 +113,9 @@ public static class Builder {
82113
private String nameNodeUri;
83114
private String basePath = "/";
84115
private Configuration configuration;
116+
private long maxClassSize = MAX_CLASS_SIZE;
117+
private int socketTimeout = DEFAULT_SOCKET_TIMEOUT;
118+
private int connectTimeout = DEFAULT_CONNECT_TIMEOUT;
85119

86120
public Builder nameNodeUri(String nameNodeUri) {
87121
this.nameNodeUri = nameNodeUri;
@@ -98,15 +132,45 @@ public Builder configuration(Configuration configuration) {
98132
return this;
99133
}
100134

135+
public Builder maxClassSize(long maxBytes) {
136+
if (maxBytes <= 0) {
137+
throw new IllegalArgumentException("maxClassSize must be positive");
138+
}
139+
this.maxClassSize = maxBytes;
140+
return this;
141+
}
142+
143+
public Builder socketTimeout(int timeoutMs) {
144+
if (timeoutMs < 0) {
145+
throw new IllegalArgumentException("socketTimeout must be >= 0");
146+
}
147+
this.socketTimeout = timeoutMs;
148+
return this;
149+
}
150+
151+
public Builder connectTimeout(int timeoutMs) {
152+
if (timeoutMs < 0) {
153+
throw new IllegalArgumentException("connectTimeout must be >= 0");
154+
}
155+
this.connectTimeout = timeoutMs;
156+
return this;
157+
}
158+
101159
public HdfsClassSource build() throws IOException {
102160
Configuration conf = configuration != null ? configuration : new Configuration();
103161

104162
if (nameNodeUri != null) {
105163
conf.set("fs.defaultFS", nameNodeUri);
106164
}
107165

166+
// Configure timeouts to prevent hanging
167+
conf.setInt("ipc.client.connect.timeout", connectTimeout);
168+
conf.setInt("ipc.client.connect.max.retries", 3);
169+
conf.setInt("ipc.ping.interval", 10000);
170+
conf.setInt("dfs.client.socket-timeout", socketTimeout);
171+
108172
FileSystem hdfs = FileSystem.get(conf);
109-
return new HdfsClassSource(hdfs, basePath);
173+
return new HdfsClassSource(hdfs, basePath, maxClassSize);
110174
}
111175
}
112176
}

0 commit comments

Comments
 (0)