Skip to content

Commit 8d17ba4

Browse files
1358445256713584452567
committed
fix: Validate sparse headers/chunks and add tests
Add comprehensive integration tests for invalid/edge sparse formats (extra fill payload, chunk total size < header, unsupported major version, extended headers on non-seekable streams, resparse part sizes and split failure). Adjust SparseFileIntegrationTests to assert the data provider interface before reading. Harden SparseReader: require seekable streams when file or chunk headers are extended, validate chunk TotalSize is at least the chunk header size, enforce FILL chunk data size == 4 (instead of allowing < or skipping extra bytes), and surface clear InvalidDataException messages. Fix SparseResparser overhead and splitting logic to use the provided maxFileSize when deciding split thresholds.
1 parent b0ac52d commit 8d17ba4

3 files changed

Lines changed: 162 additions & 17 deletions

File tree

‎FirmwareKit.Sparse.IntegrationTests/SparseFileIntegrationTests.cs‎

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,8 @@ public void GetExportStream_ReturnsSubsetOfData()
258258
Assert.Single(imported.Chunks);
259259

260260
var readBack = new byte[BlockSize];
261-
imported.Chunks[0].DataProvider.Read(0, readBack, 0, readBack.Length);
261+
var provider = Assert.IsAssignableFrom<FirmwareKit.Sparse.DataProviders.ISparseDataProvider>(imported.Chunks[0].DataProvider);
262+
provider.Read(0, readBack, 0, readBack.Length);
262263
Assert.Equal(data, readBack);
263264
}
264265

@@ -402,6 +403,128 @@ public void ValidateSparseImage_WithCrcChunk_DoesNotFailByHeaderChunkCount()
402403
}
403404
}
404405

406+
[Fact]
407+
public void FromBuffer_WhenFillChunkHasExtraPayload_ThrowsInvalidDataException()
408+
{
409+
using var sparseFile = new SparseFile(BlockSize, BlockSize);
410+
sparseFile.AddFillChunk(0x01020304, BlockSize);
411+
412+
using var output = new MemoryStream();
413+
sparseFile.WriteToStream(output, sparse: true, includeCrc: false);
414+
415+
var bytes = output.ToArray().ToList();
416+
var chunkHeaderOffset = SparseFormat.SparseHeaderSize;
417+
BinaryPrimitives.WriteUInt32LittleEndian(CollectionsMarshal.AsSpan(bytes).Slice(chunkHeaderOffset + 8, 4), SparseFormat.ChunkHeaderSize + 8u);
418+
bytes.AddRange(new byte[] { 0x55, 0x66, 0x77, 0x88 });
419+
420+
Assert.Throws<InvalidDataException>(() => SparseFile.FromBuffer(bytes.ToArray(), validateCrc: false));
421+
}
422+
423+
[Fact]
424+
public void FromBuffer_WhenChunkTotalSizeSmallerThanHeader_ThrowsInvalidDataException()
425+
{
426+
using var sparseFile = new SparseFile(BlockSize, BlockSize);
427+
sparseFile.AddRawChunk(new byte[BlockSize]);
428+
429+
using var output = new MemoryStream();
430+
sparseFile.WriteToStream(output, sparse: true, includeCrc: false);
431+
432+
var bytes = output.ToArray();
433+
var chunkHeaderOffset = SparseFormat.SparseHeaderSize;
434+
BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(chunkHeaderOffset + 8, 4), SparseFormat.ChunkHeaderSize - 1u);
435+
436+
Assert.Throws<InvalidDataException>(() => SparseFile.FromBuffer(bytes, validateCrc: false));
437+
}
438+
439+
[Fact]
440+
public void FromBuffer_WhenMajorVersionUnsupported_ThrowsInvalidDataException()
441+
{
442+
using var sparseFile = new SparseFile(BlockSize, BlockSize);
443+
sparseFile.AddRawChunk(new byte[BlockSize]);
444+
445+
using var output = new MemoryStream();
446+
sparseFile.WriteToStream(output, sparse: true, includeCrc: false);
447+
448+
var bytes = output.ToArray();
449+
BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(4, 2), 2);
450+
451+
Assert.Throws<InvalidDataException>(() => SparseFile.FromBuffer(bytes, validateCrc: false));
452+
}
453+
454+
[Fact]
455+
public void FromStream_WhenExtendedFileHeaderOnNonSeekable_ThrowsInvalidDataException()
456+
{
457+
using var sparseFile = new SparseFile(BlockSize, BlockSize);
458+
sparseFile.AddRawChunk(new byte[BlockSize]);
459+
460+
using var output = new MemoryStream();
461+
sparseFile.WriteToStream(output, sparse: true, includeCrc: false);
462+
463+
var bytes = output.ToArray();
464+
BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(8, 2), SparseFormat.SparseHeaderSize + 4);
465+
466+
using var baseStream = new MemoryStream(bytes);
467+
using var nonSeekable = new NonSeekableReadStream(baseStream);
468+
469+
Assert.Throws<InvalidDataException>(() => SparseFile.FromStream(nonSeekable, validateCrc: false));
470+
}
471+
472+
[Fact]
473+
public void FromStream_WhenExtendedChunkHeaderOnNonSeekable_ThrowsInvalidDataException()
474+
{
475+
using var sparseFile = new SparseFile(BlockSize, BlockSize);
476+
sparseFile.AddRawChunk(new byte[BlockSize]);
477+
478+
using var output = new MemoryStream();
479+
sparseFile.WriteToStream(output, sparse: true, includeCrc: false);
480+
481+
var bytes = output.ToArray();
482+
var chunkHeaderOffset = SparseFormat.SparseHeaderSize;
483+
BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(10, 2), SparseFormat.ChunkHeaderSize + 4);
484+
BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(chunkHeaderOffset + 8, 4), SparseFormat.ChunkHeaderSize + 4 + BlockSize);
485+
486+
using var baseStream = new MemoryStream(bytes);
487+
using var nonSeekable = new NonSeekableReadStream(baseStream);
488+
489+
Assert.Throws<InvalidDataException>(() => SparseFile.FromStream(nonSeekable, validateCrc: false));
490+
}
491+
492+
[Fact]
493+
public void Resparse_AllPartSparseLengths_DoNotExceedMaxFileSize()
494+
{
495+
using var sparseFile = new SparseFile(BlockSize, BlockSize * 64);
496+
for (var i = 0; i < 8; i++)
497+
{
498+
sparseFile.AddRawChunk(new byte[BlockSize * 8]);
499+
}
500+
501+
var maxFileSize = (long)SparseFormat.SparseHeaderSize + SparseFormat.ChunkHeaderSize + (BlockSize * 10);
502+
var parts = sparseFile.Resparse(maxFileSize).ToList();
503+
Assert.True(parts.Count > 1);
504+
505+
using var mergedRaw = new MemoryStream();
506+
foreach (var part in parts)
507+
{
508+
using var sparseOut = new MemoryStream();
509+
part.WriteToStream(sparseOut, sparse: true, includeCrc: false);
510+
Assert.True(sparseOut.Length <= maxFileSize, $"Part length {sparseOut.Length} exceeds max {maxFileSize}");
511+
512+
part.WriteRawToStream(mergedRaw);
513+
}
514+
515+
Assert.Equal((long)64 * BlockSize, mergedRaw.Length);
516+
}
517+
518+
[Fact]
519+
public void Resparse_WhenSingleRawChunkCannotFit_ThrowsInvalidOperationException()
520+
{
521+
using var sparseFile = new SparseFile(BlockSize, BlockSize);
522+
sparseFile.AddRawChunk(new byte[BlockSize]);
523+
524+
var tooSmall = (long)SparseFormat.SparseHeaderSize + SparseFormat.ChunkHeaderSize + BlockSize - 1;
525+
Assert.Throws<InvalidOperationException>(() => sparseFile.Resparse(tooSmall).ToList());
526+
}
527+
405528
private sealed class NonSeekableReadStream : Stream
406529
{
407530
private readonly Stream _inner;

‎FirmwareKit.Sparse/IO/SparseReader.cs‎

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,14 @@ internal static SparseFile FromStreamInternal(Stream stream, string? filePath, b
7575
throw new InvalidDataException("Invalid sparse header");
7676
}
7777

78-
if (sparseFile.Header.ChunkHeaderSize > SparseFormat.ChunkHeaderSize)
78+
if (sparseFile.Header.FileHeaderSize > SparseFormat.SparseHeaderSize)
7979
{
80-
stream.Seek(sparseFile.Header.ChunkHeaderSize - SparseFormat.ChunkHeaderSize, SeekOrigin.Current);
80+
if (!stream.CanSeek)
81+
{
82+
throw new InvalidDataException("Stream must be seekable when sparse file header is extended.");
83+
}
84+
85+
stream.Seek(sparseFile.Header.FileHeaderSize - SparseFormat.SparseHeaderSize, SeekOrigin.Current);
8186
}
8287

8388
uint? checksum = validateCrc ? Crc32.Begin() : null;
@@ -101,6 +106,11 @@ internal static SparseFile FromStreamInternal(Stream stream, string? filePath, b
101106

102107
if (sparseFile.Header.ChunkHeaderSize > SparseFormat.ChunkHeaderSize)
103108
{
109+
if (!stream.CanSeek)
110+
{
111+
throw new InvalidDataException("Stream must be seekable when chunk headers are extended.");
112+
}
113+
104114
stream.Seek(sparseFile.Header.ChunkHeaderSize - SparseFormat.ChunkHeaderSize, SeekOrigin.Current);
105115
}
106116

@@ -111,6 +121,11 @@ internal static SparseFile FromStreamInternal(Stream stream, string? filePath, b
111121
throw new InvalidDataException($"Invalid chunk header for chunk {i}: Type 0x{chunkHeader.ChunkType:X4}");
112122
}
113123

124+
if (chunkHeader.TotalSize < sparseFile.Header.ChunkHeaderSize)
125+
{
126+
throw new InvalidDataException($"Total size ({chunkHeader.TotalSize}) for chunk {i} is smaller than chunk header size ({sparseFile.Header.ChunkHeaderSize})");
127+
}
128+
114129
var dataSize = (long)chunkHeader.TotalSize - sparseFile.Header.ChunkHeaderSize;
115130
var expectedRawSize = (long)chunkHeader.ChunkSize * sparseFile.Header.BlockSize;
116131

@@ -160,9 +175,9 @@ internal static SparseFile FromStreamInternal(Stream stream, string? filePath, b
160175
break;
161176

162177
case (ushort)ChunkType.Fill:
163-
if (dataSize < 4)
178+
if (dataSize != 4)
164179
{
165-
throw new InvalidDataException($"Data size ({dataSize}) for FILL chunk {i} is less than 4 bytes");
180+
throw new InvalidDataException($"Data size ({dataSize}) for FILL chunk {i} must be 4");
166181
}
167182

168183
stream.ReadExactly(buffer4);
@@ -174,10 +189,6 @@ internal static SparseFile FromStreamInternal(Stream stream, string? filePath, b
174189
checksum = Crc32.UpdateRepeated(checksum.Value, chunk.FillValue, expectedRawSize);
175190
}
176191

177-
if (dataSize > 4)
178-
{
179-
stream.Seek(dataSize - 4, SeekOrigin.Current);
180-
}
181192
break;
182193

183194
case (ushort)ChunkType.DontCare:
@@ -291,6 +302,11 @@ internal static async Task<SparseFile> FromStreamInternalAsync(Stream stream, st
291302

292303
if (sparseFile.Header.FileHeaderSize > SparseFormat.SparseHeaderSize)
293304
{
305+
if (!stream.CanSeek)
306+
{
307+
throw new InvalidDataException("Stream must be seekable when sparse file header is extended.");
308+
}
309+
294310
stream.Seek(sparseFile.Header.FileHeaderSize - SparseFormat.SparseHeaderSize, SeekOrigin.Current);
295311
}
296312

@@ -316,6 +332,11 @@ internal static async Task<SparseFile> FromStreamInternalAsync(Stream stream, st
316332

317333
if (sparseFile.Header.ChunkHeaderSize > SparseFormat.ChunkHeaderSize)
318334
{
335+
if (!stream.CanSeek)
336+
{
337+
throw new InvalidDataException("Stream must be seekable when chunk headers are extended.");
338+
}
339+
319340
stream.Seek(sparseFile.Header.ChunkHeaderSize - SparseFormat.ChunkHeaderSize, SeekOrigin.Current);
320341
}
321342

@@ -326,6 +347,11 @@ internal static async Task<SparseFile> FromStreamInternalAsync(Stream stream, st
326347
throw new InvalidDataException($"Invalid chunk header for chunk {i}: Type 0x{chunkHeader.ChunkType:X4}");
327348
}
328349

350+
if (chunkHeader.TotalSize < sparseFile.Header.ChunkHeaderSize)
351+
{
352+
throw new InvalidDataException($"Total size ({chunkHeader.TotalSize}) for chunk {i} is smaller than chunk header size ({sparseFile.Header.ChunkHeaderSize})");
353+
}
354+
329355
var dataSize = (long)chunkHeader.TotalSize - sparseFile.Header.ChunkHeaderSize;
330356
var expectedRawSize = (long)chunkHeader.ChunkSize * sparseFile.Header.BlockSize;
331357

@@ -375,9 +401,9 @@ internal static async Task<SparseFile> FromStreamInternalAsync(Stream stream, st
375401
break;
376402

377403
case (ushort)ChunkType.Fill:
378-
if (dataSize < 4)
404+
if (dataSize != 4)
379405
{
380-
throw new InvalidDataException($"Data size ({dataSize}) for FILL chunk {i} is less than 4 bytes");
406+
throw new InvalidDataException($"Data size ({dataSize}) for FILL chunk {i} must be 4");
381407
}
382408

383409
await ReadExactlyAsync(stream, buffer4, 0, 4, cancellationToken);
@@ -389,10 +415,6 @@ internal static async Task<SparseFile> FromStreamInternalAsync(Stream stream, st
389415
checksum = Crc32.UpdateRepeated(checksum.Value, chunk.FillValue, expectedRawSize);
390416
}
391417

392-
if (dataSize > 4)
393-
{
394-
stream.Seek(dataSize - 4, SeekOrigin.Current);
395-
}
396418
break;
397419

398420
case (ushort)ChunkType.DontCare:

‎FirmwareKit.Sparse/IO/SparseResparser.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public ResparseEntry(uint startBlock, SparseChunk chunk)
2626
/// </summary>
2727
public static IEnumerable<SparseFile> Resparse(SparseFile sparseFile, long maxFileSize)
2828
{
29-
long overhead = SparseFormat.SparseHeaderSize + (2 * SparseFormat.ChunkHeaderSize) + 4;
29+
long overhead = SparseFormat.SparseHeaderSize + SparseFormat.ChunkHeaderSize;
3030

3131
if (maxFileSize <= overhead)
3232
{
@@ -77,7 +77,7 @@ public static IEnumerable<SparseFile> Resparse(SparseFile sparseFile, long maxFi
7777

7878
long currentFileLenWithHeader = fileLen + SparseFormat.ChunkHeaderSize;
7979
long availableForData = fileLimit - currentFileLenWithHeader;
80-
bool canSplit = canSplitData && (fileCurrentBlock == startBlock || availableForData > (fileLimit / 8));
80+
bool canSplit = canSplitData && (fileCurrentBlock == startBlock || availableForData > (maxFileSize / 8));
8181

8282
if (canSplit)
8383
{

0 commit comments

Comments
 (0)