Merge pull request #38 from OpenIPC/feature/ff-skip

Skip 0xFF sectors and pages in CMD_FLASH_STREAM

Author: widgetii

agent/main.c

@@ -501,13 +501,39 @@ static void handle_flash_program(const uint8_t *data, uint32_t len) {
  *   Progress: RSP_DATA [sector_done:2LE] [total:2LE] after each sector.
  *   Final: ACK_OK or ACK_CRC_ERROR.
  */
+/* Check if a 256-byte page is all 0xFF (erased state).
+ * Uses word-aligned reads for speed (~0.1µs vs 2ms page program). */
+static int page_is_ff(const uint8_t *data, uint32_t len) {
+    const uint32_t *w = (const uint32_t *)data;
+    for (uint32_t i = 0; i < len / 4; i++)
+        if (w[i] != 0xFFFFFFFF) return 0;
+    return 1;
+}
+
+/* Erase sector + program non-0xFF pages from buffer */
+static void erase_and_program(uint32_t addr, const uint8_t *buf,
+                               uint32_t bytes, uint32_t page_sz,
+                               int drain_fifo) {
+    flash_erase_sector(addr);
+    uint32_t offset = 0;
+    while (offset < bytes) {
+        uint32_t chunk = bytes - offset;
+        if (chunk > page_sz) chunk = page_sz;
+        if (!page_is_ff(&buf[offset], chunk))
+            flash_write_page(addr + offset, &buf[offset], chunk);
+        offset += chunk;
+        if (drain_fifo) proto_drain_fifo();
+    }
+}
+
 static void handle_flash_stream(const uint8_t *data, uint32_t len) {
-    if (len < 12) { proto_send_ack(ACK_CRC_ERROR); return; }
+    if (len < 44) { proto_send_ack(ACK_CRC_ERROR); return; }
     if (!flash_readable) { proto_send_ack(ACK_FLASH_ERROR); return; }
 
     uint32_t flash_addr = read_le32(&data[0]);
     uint32_t size = read_le32(&data[4]);
     uint32_t expected_crc = read_le32(&data[8]);
+    const uint8_t *bitmap = &data[12];  /* 32-byte sector bitmap */
 
     if (size == 0 || flash_addr + size > flash_info.size) {
         proto_send_ack(ACK_FLASH_ERROR);
@@ -539,6 +565,29 @@ static void handle_flash_stream(const uint8_t *data, uint32_t len) {
         uint32_t sector_bytes = size - sector_offset;
         if (sector_bytes > sector_sz) sector_bytes = sector_sz;
 
+        /* Check bitmap: bit=0 means sector is all 0xFF, skip it */
+        if (!(bitmap[s / 8] & (1 << (s % 8)))) {
+            /* Still process pending data sector if any */
+            if (pending_buf >= 0) {
+                erase_and_program(pending_addr, buf[pending_buf],
+                                  pending_bytes, page_sz, 1);
+                pending_buf = -1;
+            }
+
+            /* Erase this sector (ensure 0xFF state) but don't program */
+            flash_erase_sector(flash_addr + sector_offset);
+
+            /* Send progress — no data to receive */
+            uint8_t progress[4];
+            progress[0] = ((s + 1) >> 0) & 0xFF;
+            progress[1] = ((s + 1) >> 8) & 0xFF;
+            progress[2] = (num_sectors >> 0) & 0xFF;
+            progress[3] = (num_sectors >> 8) & 0xFF;
+            proto_send(RSP_DATA, progress, 4);
+
+            continue;  /* Don't touch rx_buf or set new pending */
+        }
+
         /* Receive sector data into rx_buf.
          * No flash operations during receive — UART stays responsive. */
         uint32_t buf_received = 0;
@@ -575,16 +624,8 @@ static void handle_flash_stream(const uint8_t *data, uint32_t len) {
         /* Process previous sector if pending (erase + program).
          * Host is streaming next sector into the OTHER buffer right now. */
         if (pending_buf >= 0) {
-            flash_erase_sector(pending_addr);
-            uint32_t offset = 0;
-            while (offset < pending_bytes) {
-                uint32_t chunk = pending_bytes - offset;
-                if (chunk > page_sz) chunk = page_sz;
-                flash_write_page(pending_addr + offset,
-                                 &buf[pending_buf][offset], chunk);
-                offset += chunk;
-                proto_drain_fifo();
-            }
+            erase_and_program(pending_addr, buf[pending_buf],
+                              pending_bytes, page_sz, 1);
         }
 
         /* This sector's buffer becomes the pending one */
@@ -598,15 +639,8 @@ static void handle_flash_stream(const uint8_t *data, uint32_t len) {
 
     /* Process the last sector (no more data to receive) */
     if (pending_buf >= 0) {
-        flash_erase_sector(pending_addr);
-        uint32_t offset = 0;
-        while (offset < pending_bytes) {
-            uint32_t chunk = pending_bytes - offset;
-            if (chunk > page_sz) chunk = page_sz;
-            flash_write_page(pending_addr + offset,
-                             &buf[pending_buf][offset], chunk);
-            offset += chunk;
-        }
+        erase_and_program(pending_addr, buf[pending_buf],
+                          pending_bytes, page_sz, 0);
     }
 
     proto_send_ack(ACK_OK);

src/defib/agent/client.py

@@ -448,8 +448,23 @@ async def write_flash(
 
         total = len(data)
         expected_crc = zlib.crc32(data) & 0xFFFFFFFF
+        sector_sz = self._sector_size or 0x10000
+        num_sectors = (total + sector_sz - 1) // sector_sz
+
+        # Build sector bitmap: bit=1 if sector has non-0xFF data
+        ff_sector = b'\xff' * sector_sz
+        bitmap = bytearray(32)
+        for s in range(num_sectors):
+            sector_data = data[s * sector_sz : (s + 1) * sector_sz]
+            if sector_data != ff_sector[:len(sector_data)]:
+                bitmap[s // 8] |= 1 << (s % 8)
+
+        skip_count = num_sectors - bin(int.from_bytes(bitmap, 'little')).count('1')
+        if skip_count > 0:
+            logger.info("Flash stream: skipping %d/%d sectors (0xFF)",
+                        skip_count, num_sectors)
 
-        payload = struct.pack("<III", addr, total, expected_crc)
+        payload = struct.pack("<III", addr, total, expected_crc) + bytes(bitmap)
         await send_packet(self._transport, CMD_FLASH_STREAM, payload)
 
         cmd, resp = await recv_response(self._transport, timeout=10.0)
@@ -459,29 +474,29 @@ async def write_flash(
 
         # Double-buffer pipeline: send sector N, get "received" signal,
         # immediately send sector N+1 while agent erases+programs N.
-        # Agent sends progress RSP_DATA after RECEIVING (not after programming),
-        # so the host can keep streaming without waiting for flash ops.
-        sector_sz = self._sector_size or 0x10000
-        num_sectors = (total + sector_sz - 1) // sector_sz
+        # Sectors with bitmap bit=0 are skipped (all 0xFF).
         offset = 0
 
         for s in range(num_sectors):
             sector_bytes = min(sector_sz, total - offset)
 
-            # Send one sector of DATA packets
-            sent = 0
-            seq = offset // WRITE_CHUNK_SIZE
-            while sent < sector_bytes:
-                chunk = min(WRITE_CHUNK_SIZE, sector_bytes - sent)
-                pkt = struct.pack("<H", seq) + data[offset + sent:offset + sent + chunk]
-                await send_packet(self._transport, RSP_DATA, pkt)
-                sent += chunk
-                seq += 1
+            if bitmap[s // 8] & (1 << (s % 8)):
+                # Send one sector of DATA packets
+                sent = 0
+                seq = offset // WRITE_CHUNK_SIZE
+                while sent < sector_bytes:
+                    chunk = min(WRITE_CHUNK_SIZE, sector_bytes - sent)
+                    pkt = struct.pack("<H", seq) + data[offset + sent:offset + sent + chunk]
+                    await send_packet(self._transport, RSP_DATA, pkt)
+                    sent += chunk
+                    seq += 1
+
+            # else: sector is all 0xFF — don't send data, agent skips it
 
             offset += sector_bytes
 
-            # Wait for "sector received" progress — agent sends this BEFORE
-            # erase+program, so we can start sending next sector immediately.
+            # Wait for progress — agent sends RSP_DATA for both data and
+            # skipped sectors (immediately for skipped, after receive for data).
             while True:
                 cmd, data_pkt = await recv_packet(self._transport, timeout=120.0)
                 if cmd == RSP_READY: