Fx natmod support & enable on Raspberry Pi Pico

.github/workflows/run-tests.yml

@@ -53,19 +53,18 @@ jobs:
         run: ./run-tests.py -j4 --print-failures
         if: failure()
         working-directory: tests
-    # Not working after MicroPython v1.23 merge.
-    # - name: Build native modules
-    #   if: matrix.test == 'all'
-    #   run: |
-    #     make -C examples/natmod/features1
-    #     make -C examples/natmod/features2
-    #     make -C examples/natmod/heapq
-    #     make -C examples/natmod/random
-    #     make -C examples/natmod/re
-    # - name: Test native modules
-    #   if: matrix.test == 'all'
-    #   run: ./run-natmodtests.py extmod/{heapq*,random*,re*}.py
-    #   working-directory: tests
+      - name: Build native modules
+        if: matrix.test == 'all'
+        run: |
+          make -C examples/natmod/features1
+          make -C examples/natmod/features2
+          make -C examples/natmod/heapq
+          make -C examples/natmod/random
+          make -C examples/natmod/re
+      - name: Test native modules
+        if: matrix.test == 'all'
+        run: ./run-natmodtests.py extmod/{heapq*,random*,re*}.py
+        working-directory: tests
 
   zephyr:
     runs-on: ubuntu-24.04

examples/natmod/.gitignore

@@ -0,0 +1 @@
+*.mpy

examples/natmod/README.md

@@ -0,0 +1,74 @@
+# Dynamic Native Modules
+
+Dynamic Native Modules are .mpy files that contain native machine code from a
+language other than Python. For more info see [the
+documentation](https://docs.micropython.org/en/latest/develop/natmod.html).
+
+This should not be confused with [User C
+Modules](https://docs.micropython.org/en/latest/develop/cmodules.html) which are
+a mechanism to add additional out-of-tree modules into the firmware build.
+
+## Examples
+
+This directory contains several examples of writing dynamic native modules, in
+two main categories:
+
+1.  Feature examples.
+
+    * `features0` - A module containing a single "factorial" function which
+      demonstrates working with integers.
+
+    * `features1` - A module that demonstrates some common tasks:
+        - defining simple functions exposed to Python
+        - defining local, helper C functions
+        - defining constant integers and strings exposed to Python
+        - getting and creating integer objects
+        - creating Python lists
+        - raising exceptions
+        - allocating memory
+        - BSS and constant data (rodata)
+        - relocated pointers in rodata
+
+    * `features2` - This is a hybrid module containing both Python and C code,
+      and additionally the C code is spread over multiple files. It also
+      demonstrates using floating point (only when the target supports
+      hardware floating point).
+
+    * `features3` - A module that shows how to use types, constant objects,
+      and creating dictionary instances.
+
+    * `features4` - A module that demonstrates how to define a class.
+
+2.  Dynamic version of existing built-ins.
+
+    This provides a way to add missing functionality to firmware that doesn't
+    include certain built-in modules. See the `heapq`, `random`, `re`,
+    `deflate`, `btree`, and `framebuf` directories.
+
+    So for example, if your firmware was compiled with `MICROPY_PY_FRAMEBUF`
+    disabled (e.g. to save flash space), then it would not include the
+    `framebuf` module. The `framebuf` native module provides a way to add the
+    `framebuf` module dynamically.
+
+    The way these work is they define a dynamic native module which
+    `#include`'s the original module and then does the necessary
+    initialisation of the module's globals dict.
+
+## Build instructions
+
+To compile an example, you need to have the same toolchain available as
+required for your target port. e.g. `arm-none-eabi-gcc` for any ARM Cortex M
+target. See the port instructions for details.
+
+You also need to have the `pyelftools` Python package available, either via
+your system package manager or installed from PyPI in a virtual environment
+with `pip`.
+
+Each example provides a Makefile. You should specify the `ARCH` argument to
+make (one of x86, x64, armv6m, armv7m, xtensa, xtensawin, rv32imc):
+
+```
+$ cd features0
+$ make ARCH=armv7m
+$ mpremote cp features0.mpy :
+```

examples/natmod/btree/Makefile

@@ -0,0 +1,38 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in btree so it can coexist)
+MOD = btree_$(ARCH)
+
+# Source files (.c or .py)
+SRC = btree_c.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc)
+ARCH = x64
+
+BTREE_DIR = $(MPY_DIR)/lib/berkeley-db-1.xx
+BERKELEY_DB_CONFIG_FILE ?= \"extmod/berkeley-db/berkeley_db_config_port.h\"
+CFLAGS += -I$(BTREE_DIR)/include
+CFLAGS += -DBERKELEY_DB_CONFIG_FILE=$(BERKELEY_DB_CONFIG_FILE)
+CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter
+
+SRC += $(addprefix $(realpath $(BTREE_DIR))/,\
+	btree/bt_close.c \
+	btree/bt_conv.c \
+	btree/bt_delete.c \
+	btree/bt_get.c \
+	btree/bt_open.c \
+	btree/bt_overflow.c \
+	btree/bt_page.c \
+	btree/bt_put.c \
+	btree/bt_search.c \
+	btree/bt_seq.c \
+	btree/bt_split.c \
+	btree/bt_utils.c \
+	mpool/mpool.c \
+	)
+
+include $(MPY_DIR)/py/dynruntime.mk
+
+# btree needs gnu99 defined
+CFLAGS += -std=gnu99

examples/natmod/btree/btree_c.c

@@ -0,0 +1,171 @@
+#define MICROPY_PY_BTREE (1)
+
+#include "py/dynruntime.h"
+
+#include <unistd.h>
+
+#if !defined(__linux__)
+void *memcpy(void *dst, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dst, src, n);
+}
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+void *memmove(void *dest, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dest, src, n);
+}
+
+void *malloc(size_t n) {
+    void *ptr = m_malloc(n);
+    return ptr;
+}
+void *realloc(void *ptr, size_t n) {
+    mp_printf(&mp_plat_print, "UNDEF %d\n", __LINE__);
+    return NULL;
+}
+void *calloc(size_t n, size_t m) {
+    void *ptr = m_malloc(n * m);
+    // memory already cleared by conservative GC
+    return ptr;
+}
+
+void free(void *ptr) {
+    m_free(ptr);
+}
+
+void abort_(void) {
+    nlr_raise(mp_obj_new_exception(mp_load_global(MP_QSTR_RuntimeError)));
+}
+
+int puts(const char *s) {
+    return mp_printf(&mp_plat_print, "%s\n", s);
+}
+
+int native_errno;
+#if defined(__linux__)
+int *__errno_location (void)
+#else
+int *__errno (void)
+#endif
+{
+    return &native_errno;
+}
+
+ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol);
+    mp_uint_t out_sz = stream_p->write(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
+    if (out_sz == MP_STREAM_ERROR) {
+        return -1;
+    } else {
+        return out_sz;
+    }
+}
+
+ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol);
+    mp_uint_t out_sz = stream_p->read(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
+    if (out_sz == MP_STREAM_ERROR) {
+        return -1;
+    } else {
+        return out_sz;
+    }
+}
+
+off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) {
+    const mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol);
+    struct mp_stream_seek_t seek_s;
+    seek_s.offset = offset;
+    seek_s.whence = whence;
+    mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &native_errno);
+    if (res == MP_STREAM_ERROR) {
+        return -1;
+    }
+    return seek_s.offset;
+}
+
+int mp_stream_posix_fsync(void *stream) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol);
+    mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_FLUSH, 0, &native_errno);
+    if (res == MP_STREAM_ERROR) {
+        return -1;
+    }
+    return res;
+}
+
+mp_obj_full_type_t btree_type;
+mp_getiter_iternext_custom_t btree_getiter_iternext;
+
+#include "extmod/modbtree.c"
+
+mp_map_elem_t btree_locals_dict_table[8];
+static MP_DEFINE_CONST_DICT(btree_locals_dict, btree_locals_dict_table);
+
+static mp_obj_t btree_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    // The allowed_args array must have its qstr's populated at runtime.
+    enum { ARG_flags, ARG_cachesize, ARG_pagesize, ARG_minkeypage };
+    mp_arg_t allowed_args[] = {
+        { MP_QSTR_, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} },
+        { MP_QSTR_, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} },
+        { MP_QSTR_, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} },
+        { MP_QSTR_, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} },
+    };
+    allowed_args[0].qst = MP_QSTR_flags;
+    allowed_args[1].qst = MP_QSTR_cachesize;
+    allowed_args[2].qst = MP_QSTR_pagesize;
+    allowed_args[3].qst = MP_QSTR_minkeypage;
+
+    // Make sure we got a stream object
+    mp_get_stream_raise(pos_args[0], MP_STREAM_OP_READ | MP_STREAM_OP_WRITE | MP_STREAM_OP_IOCTL);
+
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    BTREEINFO openinfo = {0};
+    openinfo.flags = args[ARG_flags].u_int;
+    openinfo.cachesize = args[ARG_cachesize].u_int;
+    openinfo.psize = args[ARG_pagesize].u_int;
+    openinfo.minkeypage = args[ARG_minkeypage].u_int;
+    DB *db = __bt_open(MP_OBJ_TO_PTR(pos_args[0]), &btree_stream_fvtable, &openinfo, 0);
+    if (db == NULL) {
+        mp_raise_OSError(native_errno);
+    }
+
+    return MP_OBJ_FROM_PTR(btree_new(db, pos_args[0]));
+}
+static MP_DEFINE_CONST_FUN_OBJ_KW(btree_open_obj, 1, btree_open);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    btree_getiter_iternext.getiter = btree_getiter;
+    btree_getiter_iternext.iternext = btree_iternext;
+
+    btree_type.base.type = (void*)&mp_fun_table.type_type;
+    btree_type.flags = MP_TYPE_FLAG_ITER_IS_CUSTOM;
+    btree_type.name = MP_QSTR_btree;
+    MP_OBJ_TYPE_SET_SLOT(&btree_type, print, btree_print, 0);
+    MP_OBJ_TYPE_SET_SLOT(&btree_type, iter, &btree_getiter_iternext, 1);
+    MP_OBJ_TYPE_SET_SLOT(&btree_type, binary_op, btree_binary_op, 2);
+    MP_OBJ_TYPE_SET_SLOT(&btree_type, subscr, btree_subscr, 3);
+    btree_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&btree_close_obj) };
+    btree_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_flush), MP_OBJ_FROM_PTR(&btree_flush_obj) };
+    btree_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get), MP_OBJ_FROM_PTR(&btree_get_obj) };
+    btree_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_put), MP_OBJ_FROM_PTR(&btree_put_obj) };
+    btree_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_seq), MP_OBJ_FROM_PTR(&btree_seq_obj) };
+    btree_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_keys), MP_OBJ_FROM_PTR(&btree_keys_obj) };
+    btree_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_values), MP_OBJ_FROM_PTR(&btree_values_obj) };
+    btree_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_items), MP_OBJ_FROM_PTR(&btree_items_obj) };
+    MP_OBJ_TYPE_SET_SLOT(&btree_type, locals_dict, (void*)&btree_locals_dict, 4);
+
+    mp_store_global(MP_QSTR_open, MP_OBJ_FROM_PTR(&btree_open_obj));
+    mp_store_global(MP_QSTR_INCL, MP_OBJ_NEW_SMALL_INT(FLAG_END_KEY_INCL));
+    mp_store_global(MP_QSTR_DESC, MP_OBJ_NEW_SMALL_INT(FLAG_DESC));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/deflate/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in uzlib so it can coexist)
+MOD = deflate_$(ARCH)
+
+# Source files (.c or .py)
+SRC = deflate.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk