fix: report proper sketch and loader sizes in CI packaging

Author: pillo79

.github/workflows/package_core.yml

@@ -117,9 +117,15 @@ jobs:
           grep -i "warning:" $REPORT.stdout > $REPORT.warnings || true
 
           # extract the memory usage table (from the header to the first non-% line)
+          # override the size of the Flash with the size of the loader partition
+          # and add the size of the sketch partition (not reported by Zephyr)
+          LOADER_SIZE=$(( $(cat variants/${{ matrix.variant }}/syms-static.ld | grep '_loader_max_size' | cut -d '=' -f 2 | tr -d ') ;') ))
+          SKETCH_SIZE=$(( $(cat variants/${{ matrix.variant }}/syms-static.ld | grep '_sketch_max_size' | cut -d '=' -f 2 | tr -d ') ;') ))
           cat $REPORT.stdout | sed -n '/^Memory region/,/^[^%]*$/p' | head -n -1 \
                              | awk 'BEGIN {split("B KB MB GB", u); for(i in u) m[u[i]]=1024^(i-1)} /:/ {print "[\"" $1 "\"," $2*m[$3] "," $4*m[$5] "]"}' \
-                             | sort | jq -s > $REPORT.meminfo
+                             | sort \
+                             | jq -s "map((select(.[0] == \"FLASH:\") | .[2]) |= ${LOADER_SIZE}) + [ [ \"SKETCH:\", 0, ${SKETCH_SIZE} ] ]" > $REPORT.meminfo
+          jq
 
       - name: Package board artifacts
         if: ${{ !cancelled() }}

extra/build.sh

@@ -106,6 +106,7 @@ extra/gen_provides.py "${BUILD_DIR}/zephyr/zephyr.elf" -LF \
 	"+kheap__system_heap" \
 	"*sketch_base_addr=_sketch_start" \
 	"*sketch_max_size=_sketch_max_size" \
+	"*loader_max_size=_loader_max_size" \
 	"malloc=__wrap_malloc" \
 	"free=__wrap_free" \
 	"realloc=__wrap_realloc" \

extra/ci_inspect_logs.py

@@ -412,6 +412,7 @@ def print_mem_report(artifact, artifact_boards):
     f_print("<th rowspan='2'>SoC</th>", end='')
     f_print("<th rowspan='2'>FLASH</th>", end='')
     f_print("<th rowspan='2'>RAM</th>", end='')
+    f_print("<th rowspan='2'>User<br>sketch</th>", end='')
     f_print("<th colspan='2'>User heaps</th>", end='')
     f_print("<th colspan='2'>OS heaps</th>", end='')
     f_print("</tr>")
@@ -428,14 +429,15 @@ def print_mem_report(artifact, artifact_boards):
                 f"<code>{soc}</code>",
                 color_entry(BOARD_LOADERS[board].meminfo['FLASH']),
                 color_entry(BOARD_LOADERS[board].meminfo['RAM']),
+                f"${{{ BOARD_LOADERS[board].meminfo['SKETCH'][1] }}}$",
                 f"${{{ BOARD_LOADERS[board].config.get('CONFIG_HEAP_MEM_POOL_SIZE', 0) }}}$",
                 f"${{{ BOARD_LOADERS[board].config['CONFIG_SRAM_SIZE']*1024 - BOARD_LOADERS[board].meminfo['RAM'][0] }}}$",
                 f"${{{ BOARD_LOADERS[board].config['CONFIG_LLEXT_HEAP_SIZE']*1024 }}}$",
                 f"${{{ BOARD_LOADERS[board].config.get('CONFIG_MBEDTLS_HEAP_SIZE', '-') }}}$"
               ]
 
         f_print("<tr>")
-        col_aligns = ['center', 'left', 'center', 'right', 'right', 'right', 'right', 'right', 'right']
+        col_aligns = ['center', 'left', 'center', 'right', 'right', 'right', 'right', 'right', 'right', 'right']
         for index, cell in enumerate(row):
             f_print(f"<td align='{col_aligns[index]}'>\n\n{cell}\n\n</td>")
         f_print("</tr>")
@@ -444,7 +446,7 @@ def print_mem_report(artifact, artifact_boards):
     extra_data_present = False
     for soc in sorted(list(set([ ALL_BOARD_DATA[board]['soc'] for board in artifact_boards ]))):
         soc_boards = [ board for board in artifact_boards if ALL_BOARD_DATA[board]['soc'] == soc ]
-        sorted_regions = sorted(r for r in REGIONS_BY_SOC[soc] if r not in ('FLASH', 'RAM'))
+        sorted_regions = sorted(r for r in REGIONS_BY_SOC[soc] if r not in ('FLASH', 'RAM', 'SKETCH'))
         if not sorted_regions:
             continue
 

extra/package_core.sh

@@ -49,14 +49,24 @@ for board in $EXCLUDED_BOARDS ; do
 	# remove (even commented) lines for excluded boards
 	sed -i "/^\(\s*#\s*\)\?${board}\./d" $TEMP_BOARDS
 done
+# set proper maximum sizes for included variants
+for variant in $INCLUDED_VARIANTS ; do
+	board=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.variant == \"${variant}\")) | .[0].board")
+	# maximum sketch size: size of sketch partition (exact limit)
+	# maximum data size: configured LLEXT heap size (larger bound, real limit is smaller)
+	CODE_SIZE=$(( $(cat variants/${variant}/syms-static.ld | grep '_sketch_max_size' | cut -d '=' -f 2 | tr -d ');') ))
+	DATA_SIZE=$(( 1024*$(cat firmwares/zephyr-${variant}.config | grep 'LLEXT_HEAP_SIZE' | cut -d '=' -f 2) ))
+	sed -i -e "s/^${board}\.upload\.maximum_size=.*/${board}.upload.maximum_size=${CODE_SIZE}/" $TEMP_BOARDS
+	sed -i -e "s/^${board}\.upload\.maximum_data_size=.*/${board}.upload.maximum_data_size=${DATA_SIZE}/" $TEMP_BOARDS
+done
 # remove multiple empty lines
 sed -i '/^$/N;/^\n$/D' $TEMP_BOARDS
 
 # create a temporary platform.txt file with the correct version
 TEMP_PLATFORM=$(mktemp -p . | sed 's/\.\///')
 cat platform.txt > ${TEMP_PLATFORM}
 [ -z "$ARTIFACT_NAME" ] || sed -ie "s/^name=.*/name=${ARTIFACT_NAME}/" ${TEMP_PLATFORM}
-sed -ie "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM}
+sed -i -e "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM}
 
 declutter_file() {
 	# remove comments, whitespace at EOL, '/' dir terminators and empty lines

loader/main.c

@@ -23,6 +23,8 @@ LOG_MODULE_REGISTER(sketch);
 #include <zephyr/drivers/uart.h>
 #include <zephyr/usb/usb_device.h>
 
+#include <zephyr/devicetree/fixed-partitions.h>
+
 #define HEADER_LEN 16
 
 struct sketch_header_v1 {
@@ -89,10 +91,23 @@ void llext_entry(void *arg0, void *arg1, void *arg2) {
 }
 #endif /* CONFIG_USERSPACE */
 
+/* Export Flash parameters for use by core building scripts */
 __attribute__((retain)) const uintptr_t sketch_base_addr =
 	DT_REG_ADDR(DT_GPARENT(DT_NODELABEL(user_sketch))) + DT_REG_ADDR(DT_NODELABEL(user_sketch));
 __attribute__((retain)) const uintptr_t sketch_max_size = DT_REG_SIZE(DT_NODELABEL(user_sketch));
 
+#if DT_HAS_FIXED_PARTITION_LABEL(image_0)
+ #define LOADER_MAX_SIZE DT_REG_SIZE(DT_NODE_BY_FIXED_PARTITION_LABEL(image_0))
+#elif CONFIG_FLASH_LOAD_SIZE > 0
+ #define LOADER_MAX_SIZE CONFIG_FLASH_LOAD_SIZE
+#else
+ #ifndef CONFIG_FLASH_LOAD_OFFSET
+  #define CONFIG_FLASH_LOAD_OFFSET 0
+ #endif
+ #define LOADER_MAX_SIZE (DT_REG_SIZE(DT_NODELABEL(flash0)) - CONFIG_FLASH_LOAD_OFFSET)
+#endif
+__attribute__((retain)) const uintptr_t loader_max_size = LOADER_MAX_SIZE;
+
 static int loader(const struct shell *sh) {
 	const struct flash_area *fa;
 	int rc;