Improved SECV format

removed the need for the index table

Author: Wavesonics

app/src/main/kotlin/com/darkrockstudios/app/securecamera/security/streaming/ChunkedStreamingDecryptor.kt

@@ -13,8 +13,8 @@ import javax.crypto.spec.SecretKeySpec
 
 /**
  * Implementation of StreamingDecryptor that decrypts SECV files chunk by chunk.
- * Supports random access for video seeking by using the chunk index table.
- * Reads the trailer format with trailer and index at the end of the file.
+ * Supports random access for video seeking by calculating chunk offsets arithmetically.
+ * Reads the header format with metadata at the start of the file.
  *
  * Thread-safe via mutex for all read operations.
  */
@@ -27,48 +27,31 @@ class ChunkedStreamingDecryptor(
 	private val randomAccessFile: RandomAccessFile
 	private var isClosed = false
 
-	private val trailer: SecvFileFormat.SecvTrailer
-	private val chunkIndex: List<SecvFileFormat.ChunkIndexEntry>
+	private val header: SecvFileFormat.SecvHeader
 
 	// Cache for the most recently decrypted chunk to avoid re-decryption on sequential reads
 	private var cachedChunkIndex: Long = -1
 	private var cachedChunkData: ByteArray? = null
 
 	override val totalSize: Long
-		get() = trailer.originalSize
+		get() = header.originalSize
 
 	override val chunkSize: Int
-		get() = trailer.chunkSize
+		get() = header.chunkSize
 
 	init {
 		require(encryptedFile.exists()) { "Encrypted file does not exist: ${encryptedFile.absolutePath}" }
 
 		randomAccessFile = RandomAccessFile(encryptedFile, "r") ?: error("Failed to open file for reading")
-		val fileLength = randomAccessFile.length()
 
-		// Read trailer from end of file
-		val trailerPosition = SecvFileFormat.calculateTrailerPosition(fileLength)
-		randomAccessFile.seek(trailerPosition)
-		val trailerBytes = ByteArray(SecvFileFormat.TRAILER_SIZE)
-		randomAccessFile.readFully(trailerBytes)
-		trailer = SecvFileFormat.SecvTrailer.fromByteArray(trailerBytes)
+		// Read header from start of file
+		randomAccessFile.seek(0)
+		val headerBytes = ByteArray(SecvFileFormat.HEADER_SIZE)
+		randomAccessFile.readFully(headerBytes)
+		header = SecvFileFormat.SecvHeader.fromByteArray(headerBytes)
 
-		require(trailer.version == SecvFileFormat.VERSION) {
-			"Unsupported SECV version: ${trailer.version}"
-		}
-
-		// Read chunk index table (just before trailer)
-		val indexPosition = SecvFileFormat.calculateIndexTablePosition(fileLength, trailer.totalChunks)
-		randomAccessFile.seek(indexPosition)
-		val indexTableSize = trailer.totalChunks * SecvFileFormat.CHUNK_INDEX_ENTRY_SIZE
-		val indexBytes = ByteArray(indexTableSize.toInt())
-		randomAccessFile.readFully(indexBytes)
-
-		chunkIndex = (0 until trailer.totalChunks).map { i ->
-			SecvFileFormat.ChunkIndexEntry.fromByteArray(
-				indexBytes,
-				(i * SecvFileFormat.CHUNK_INDEX_ENTRY_SIZE).toInt()
-			)
+		require(header.version == SecvFileFormat.VERSION) {
+			"Unsupported SECV version: ${header.version}"
 		}
 	}
 
@@ -78,6 +61,11 @@ class ChunkedStreamingDecryptor(
 		require(length >= 0) { "Length must be non-negative" }
 		require(offset + length <= buffer.size) { "Offset + length exceeds buffer size" }
 
+		// Reading 0 bytes always succeeds and returns 0
+		if (length == 0) {
+			return 0
+		}
+
 		if (position >= totalSize) {
 			return -1 // EOF
 		}
@@ -138,13 +126,22 @@ class ChunkedStreamingDecryptor(
 	private suspend fun decryptChunk(chunkIdx: Long): ByteArray = withContext(Dispatchers.IO) {
 		val raf = randomAccessFile
 
-		require(chunkIdx < chunkIndex.size) { "Chunk index out of bounds: $chunkIdx" }
+		require(chunkIdx < header.totalChunks) { "Chunk index out of bounds: $chunkIdx" }
+
+		// Calculate chunk offset arithmetically
+		val chunkOffset = SecvFileFormat.calculateChunkOffset(chunkIdx, header.chunkSize)
 
-		val entry = chunkIndex[chunkIdx.toInt()]
+		// Determine encrypted size based on whether this is the final chunk
+		val isFinalChunk = (chunkIdx == header.totalChunks - 1)
+		val encryptedSize = if (isFinalChunk) {
+			SecvFileFormat.calculateEncryptedChunkSize(header.finalChunkPlaintextSize)
+		} else {
+			SecvFileFormat.calculateFullEncryptedChunkSize(header.chunkSize)
+		}
 
 		// Read the encrypted chunk data (IV + ciphertext with auth tag)
-		val encryptedData = ByteArray(entry.encryptedSize)
-		raf.seek(entry.offset)
+		val encryptedData = ByteArray(encryptedSize)
+		raf.seek(chunkOffset)
 		raf.readFully(encryptedData)
 
 		val iv = encryptedData.copyOfRange(0, SecvFileFormat.IV_SIZE)

app/src/main/kotlin/com/darkrockstudios/app/securecamera/security/streaming/ChunkedStreamingEncryptor.kt

@@ -16,10 +16,10 @@ import javax.crypto.spec.SecretKeySpec
 /**
  * Implementation of StreamingEncryptor that encrypts data in chunks using AES-GCM.
  *
- * This encryptor writes data in the SECV trailer format:
- * 1. Encrypted chunks are written sequentially as data arrives
- * 2. On close, the index table and trailer are appended to the end
- * 3. No file rewriting needed, preventing memory spikes on large videos
+ * This encryptor writes data in the SECV header format:
+ * 1. Writes 64-byte placeholder header at start
+ * 2. Encrypted chunks are written sequentially as data arrives
+ * 3. On close, seeks to position 0 and writes final header with metadata
  *
  * Thread-safe via mutex for all write operations.
  */
@@ -36,7 +36,8 @@ class ChunkedStreamingEncryptor(
 	private var currentBuffer = ByteArray(chunkSize)
 	private var bufferPosition = 0
 	private var totalBytesWritten = 0L
-	private val chunkIndexEntries = mutableListOf<SecvFileFormat.ChunkIndexEntry>()
+	private var totalChunks = 0L
+	private var finalChunkPlaintextSize = 0
 	private var isClosed = false
 	private var isFlushed = false
 
@@ -48,9 +49,13 @@ class ChunkedStreamingEncryptor(
 
 		randomAccessFile = RandomAccessFile(outputFile, "rw")
 
-		// Chunks are written starting at offset 0
-		// Index table and trailer will be appended at the end
-		currentChunkDataOffset = 0L
+		// Write placeholder header (64 zero bytes) at start
+		// Will be filled in with actual metadata on close()
+		val placeholderHeader = ByteArray(SecvFileFormat.HEADER_SIZE)
+		randomAccessFile?.write(placeholderHeader)
+
+		// Chunks are written starting after the header
+		currentChunkDataOffset = SecvFileFormat.HEADER_SIZE.toLong()
 	}
 
 	override suspend fun write(data: ByteArray, offset: Int, length: Int) {
@@ -101,7 +106,7 @@ class ChunkedStreamingEncryptor(
 	}
 
 	/**
-	 * Encrypts a chunk of data and writes it to the temporary storage.
+	 * Encrypts a chunk of data and writes it to the file.
 	 * Must be called while holding the mutex.
 	 */
 	private suspend fun writeChunk(data: ByteArray, length: Int) = withContext(Dispatchers.IO) {
@@ -122,20 +127,16 @@ class ChunkedStreamingEncryptor(
 			val plaintext = if (length == data.size) data else data.copyOf(length)
 			val ciphertext = cipher.doFinal(plaintext)
 
-			// Record the chunk index entry
-			val encryptedSize = SecvFileFormat.IV_SIZE + ciphertext.size
-			chunkIndexEntries.add(
-				SecvFileFormat.ChunkIndexEntry(
-					offset = currentChunkDataOffset,
-					encryptedSize = encryptedSize
-				)
-			)
+			// Track chunk count and final chunk plaintext size
+			totalChunks++
+			finalChunkPlaintextSize = length
 
 			// Write IV + ciphertext (which includes auth tag)
 			raf.seek(currentChunkDataOffset)
 			raf.write(iv)
 			raf.write(ciphertext)
 
+			val encryptedSize = SecvFileFormat.IV_SIZE + ciphertext.size
 			currentChunkDataOffset += encryptedSize
 		} finally {
 			// Immediately zero key bytes after use
@@ -164,17 +165,15 @@ class ChunkedStreamingEncryptor(
 				val plaintext = currentBuffer.copyOf(bufferPosition)
 				val ciphertext = cipher.doFinal(plaintext)
 
-				val encryptedSize = SecvFileFormat.IV_SIZE + ciphertext.size
-				chunkIndexEntries.add(
-					SecvFileFormat.ChunkIndexEntry(
-						offset = currentChunkDataOffset,
-						encryptedSize = encryptedSize
-					)
-				)
+				// Track chunk count and final chunk plaintext size
+				totalChunks++
+				finalChunkPlaintextSize = bufferPosition
 
 				raf.seek(currentChunkDataOffset)
 				raf.write(iv)
 				raf.write(ciphertext)
+
+				val encryptedSize = SecvFileFormat.IV_SIZE + ciphertext.size
 				currentChunkDataOffset += encryptedSize
 				bufferPosition = 0
 			} finally {
@@ -183,19 +182,16 @@ class ChunkedStreamingEncryptor(
 			}
 		}
 
-		// Append index table at current position
-		for (entry in chunkIndexEntries) {
-			raf.write(entry.toByteArray())
-		}
-
-		// Append trailer at end (now we know totalChunks and originalSize)
-		val trailer = SecvFileFormat.SecvTrailer(
+		// Seek to beginning and write header with final metadata
+		raf.seek(0)
+		val header = SecvFileFormat.SecvHeader(
 			version = SecvFileFormat.VERSION,
 			chunkSize = chunkSize,
-			totalChunks = chunkIndexEntries.size.toLong(),
-			originalSize = totalBytesWritten
+			totalChunks = totalChunks,
+			originalSize = totalBytesWritten,
+			finalChunkPlaintextSize = finalChunkPlaintextSize
 		)
-		raf.write(trailer.toByteArray())
+		raf.write(header.toByteArray())
 
 		randomAccessFile?.close()
 		randomAccessFile = null

app/src/main/kotlin/com/darkrockstudios/app/securecamera/security/streaming/SecvFileFormat.kt

@@ -6,55 +6,52 @@ import java.nio.ByteOrder
 /**
  * Constants and utilities for the SECV (Secure Encrypted Camera Video) file format.
  *
- * File Format:
- * [Encrypted Chunks]
- *   - Per chunk: [12-byte IV][ciphertext][16-byte auth tag]
- *
- * [Chunk Index Table: 12 bytes per chunk]
- *   - Chunk offset: uint64 (8 bytes)
- *   - Encrypted size: uint32 (4 bytes)
- *
- * [Trailer: 64 bytes] - Located at end of file
+ * File Format (Version 1):
+ * [Header: 64 bytes] - Located at start of file
  *   - Magic: "SECV" (4 bytes)
  *   - Version: uint16 (2 bytes)
  *   - Chunk size: uint32 (4 bytes)
  *   - Total chunks: uint64 (8 bytes)
  *   - Original size: uint64 (8 bytes)
- *   - Reserved: padding to 64 bytes (38 bytes)
+ *   - Final chunk plaintext size: uint32 (4 bytes)
+ *   - Reserved: padding to 64 bytes (34 bytes)
  *
- * The trailer format (chunks first, metadata at end) eliminates the need
- * to rewrite the entire file when encryption completes, preventing memory
- * spikes from loading large videos into RAM.
+ * [Encrypted Chunks]
+ *   - Per chunk: [12-byte IV][ciphertext][16-byte auth tag]
+ *
+ * Design Rationale:
+ * - Chunk offsets calculated arithmetically: offset = 64 + (chunkIndex * (chunkSize + 28))
  */
 object SecvFileFormat {
 	const val MAGIC = "SECV"
 	const val VERSION: Short = 1
-	const val TRAILER_SIZE = 64
-	const val CHUNK_INDEX_ENTRY_SIZE = 12
+	const val HEADER_SIZE = 64
 	const val IV_SIZE = 12
 	const val AUTH_TAG_SIZE = 16
 	const val DEFAULT_CHUNK_SIZE = 1_048_576 // 1MB
 
 	const val FILE_EXTENSION = "secv"
 
-	// Trailer field offsets
+	// Header field offsets
 	private const val OFFSET_MAGIC = 0
 	private const val OFFSET_VERSION = 4
 	private const val OFFSET_CHUNK_SIZE = 6
 	private const val OFFSET_TOTAL_CHUNKS = 10
 	private const val OFFSET_ORIGINAL_SIZE = 18
+	private const val OFFSET_FINAL_CHUNK_SIZE = 26
 
 	/**
-	 * Represents the trailer of a SECV file (metadata at end of file).
+	 * Represents the header of a SECV file (metadata at start of file).
 	 */
-	data class SecvTrailer(
+	data class SecvHeader(
 		val version: Short,
 		val chunkSize: Int,
 		val totalChunks: Long,
-		val originalSize: Long
+		val originalSize: Long,
+		val finalChunkPlaintextSize: Int
 	) {
 		fun toByteArray(): ByteArray {
-			val buffer = ByteBuffer.allocate(TRAILER_SIZE)
+			val buffer = ByteBuffer.allocate(HEADER_SIZE)
 			buffer.order(ByteOrder.LITTLE_ENDIAN)
 
 			// Magic
@@ -67,14 +64,16 @@ object SecvFileFormat {
 			buffer.putLong(totalChunks)
 			// Original size
 			buffer.putLong(originalSize)
+			// Final chunk plaintext size
+			buffer.putInt(finalChunkPlaintextSize)
 			// Reserved (remaining bytes are zero by default)
 
 			return buffer.array()
 		}
 
 		companion object {
-			fun fromByteArray(bytes: ByteArray): SecvTrailer {
-				require(bytes.size >= TRAILER_SIZE) { "Trailer too small" }
+			fun fromByteArray(bytes: ByteArray): SecvHeader {
+				require(bytes.size >= HEADER_SIZE) { "Header too small" }
 
 				val buffer = ByteBuffer.wrap(bytes)
 				buffer.order(ByteOrder.LITTLE_ENDIAN)
@@ -88,39 +87,14 @@ object SecvFileFormat {
 				val chunkSize = buffer.int
 				val totalChunks = buffer.long
 				val originalSize = buffer.long
+				val finalChunkPlaintextSize = buffer.int
 
-				return SecvTrailer(
+				return SecvHeader(
 					version = version,
 					chunkSize = chunkSize,
 					totalChunks = totalChunks,
-					originalSize = originalSize
-				)
-			}
-		}
-	}
-
-	/**
-	 * Represents an entry in the chunk index table.
-	 */
-	data class ChunkIndexEntry(
-		val offset: Long,
-		val encryptedSize: Int
-	) {
-		fun toByteArray(): ByteArray {
-			val buffer = ByteBuffer.allocate(CHUNK_INDEX_ENTRY_SIZE)
-			buffer.order(ByteOrder.LITTLE_ENDIAN)
-			buffer.putLong(offset)
-			buffer.putInt(encryptedSize)
-			return buffer.array()
-		}
-
-		companion object {
-			fun fromByteArray(bytes: ByteArray, offset: Int = 0): ChunkIndexEntry {
-				val buffer = ByteBuffer.wrap(bytes, offset, CHUNK_INDEX_ENTRY_SIZE)
-				buffer.order(ByteOrder.LITTLE_ENDIAN)
-				return ChunkIndexEntry(
-					offset = buffer.long,
-					encryptedSize = buffer.int
+					originalSize = originalSize,
+					finalChunkPlaintextSize = finalChunkPlaintextSize
 				)
 			}
 		}
@@ -135,19 +109,19 @@ object SecvFileFormat {
 	}
 
 	/**
-	 * Calculate the position of the trailer in the file (last 64 bytes).
-	 * For trailer format, trailer is at: fileLength - TRAILER_SIZE
+	 * Calculate the size of a full encrypted chunk.
+	 * Full chunk size = IV (12 bytes) + chunkSize + auth tag (16 bytes) = chunkSize + 28
 	 */
-	fun calculateTrailerPosition(fileLength: Long): Long {
-		return fileLength - TRAILER_SIZE
+	fun calculateFullEncryptedChunkSize(chunkSize: Int): Int {
+		return chunkSize + IV_SIZE + AUTH_TAG_SIZE
 	}
 
 	/**
-	 * Calculate the position of the index table in the file.
-	 * For trailer format, index is at: fileLength - TRAILER_SIZE - (totalChunks * CHUNK_INDEX_ENTRY_SIZE)
+	 * Calculate the file offset for a given chunk index.
+	 * Offset = header (64 bytes) + (chunkIndex * full encrypted chunk size)
 	 */
-	fun calculateIndexTablePosition(fileLength: Long, totalChunks: Long): Long {
-		return fileLength - TRAILER_SIZE - (totalChunks * CHUNK_INDEX_ENTRY_SIZE)
+	fun calculateChunkOffset(chunkIndex: Long, chunkSize: Int): Long {
+		return HEADER_SIZE + (chunkIndex * calculateFullEncryptedChunkSize(chunkSize))
 	}
 
 	/**