From c4d353ebe3fcdb05cb0e04578039b5282a5dee08 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 23 Oct 2025 12:42:00 +0100 Subject: [PATCH 1/2] vpages: Add a simple virtual continuous page allocator Add a simple virtual page allocator that uses the Zephyr memory mapping infrastructure to allocate pages from a virtual region and map them to physical pages. Due to simplicity, the number of allocations is limited to CONFIG_SOF_VPAGE_MAX_ALLOCS Signed-off-by: Liam Girdwood --- zephyr/Kconfig | 10 + zephyr/include/sof/lib/vpages.h | 49 +++++ zephyr/lib/vpages.c | 344 ++++++++++++++++++++++++++++++++ 3 files changed, 403 insertions(+) create mode 100644 zephyr/include/sof/lib/vpages.h create mode 100644 zephyr/lib/vpages.c diff --git a/zephyr/Kconfig b/zephyr/Kconfig index a642ee52471c..3d7e8b3943d7 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -138,6 +138,16 @@ config SOF_USERSPACE_APPLICATION Not manually settable. This is effectively a shortcut to replace numerous checks for (CONFIG_USERSPACE && !CONFIG_SOF_USERSPACE_PROXY) +config SOF_VPAGE_MAX_ALLOCS + int "Number of virtual memory page allocation elements" + default 128 + help + This setting defines the maximum number of virtual memory page allocation + elements that can be tracked. Each allocation element represents a + contiguous block of virtual memory allocated from the virtual memory + region. Increasing this number allows for more simultaneous page allocations, + but also increases the memory overhead for tracking these allocations. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" help diff --git a/zephyr/include/sof/lib/vpages.h b/zephyr/include/sof/lib/vpages.h new file mode 100644 index 000000000000..a360c6bf3a8a --- /dev/null +++ b/zephyr/include/sof/lib/vpages.h @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Virtual Page Allocator API */ +#ifndef __SOF_LIB_VPAGE_H__ +#define __SOF_LIB_VPAGE_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Allocate virtual pages + * Allocates a specified number of contiguous virtual memory pages by mapping + * physical pages. + * + * @param[in] pages Number of 4kB pages to allocate. + * + * @return Pointer to the allocated virtual memory region, or NULL on failure. + */ +void *alloc_vpages(uint32_t pages); + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param[in] ptr Pointer to the memory pages to free. + */ +void free_vpages(void *ptr); + +/** + * @brief Initialize virtual page allocator + * + * Initializes a virtual page allocator that manages a virtual memory region + * using a page table and block structures. + * + * @retval 0 if successful. + * @retval -ENOMEM on creation failure. + */ +int init_vpages(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VPAGE_H__ */ diff --git a/zephyr/lib/vpages.c b/zephyr/lib/vpages.c new file mode 100644 index 000000000000..fcd7eef3c987 --- /dev/null +++ b/zephyr/lib/vpages.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vpage, CONFIG_SOF_LOG_LEVEL); + +/* Simple Page Allocator. + * + * This allocator manages the allocation and deallocation of virtual memory pages from + * a predefined virtual memory region which is larger than the physical memory region. + * + * Both memory regions are divided into 4kB pages that are represented as blocks in a + * bitmap using the zephyr sys_mem_blocks API. The virtual block map tracks the allocation + * of virtual memory pages while the physical block map in the Zephyr MM driver tracks + * the allocation of physical memory pages. + */ + +/* max number of allocation elements */ +#define VPAGE_MAX_ALLOCS CONFIG_SOF_VPAGE_MAX_ALLOCS + +/* + * Virtual memory allocation element - tracks allocated virtual page id and size + */ +struct valloc_elem { + uint16_t pages; /* number of 4kB pages allocated in contiguous block */ + uint16_t vpage; /* virtual page number from start of region */ +} __packed; + +/* + * Virtual page table structure + * + * This structure holds all information about virtual memory pages + * including the number of free and total pages, the virtual memory + * region, the block allocator for virtual pages and the allocation + * elements. + */ +struct vpage_context { + struct k_mutex lock; + uint32_t free_pages; /* number of free 4kB pages */ + uint32_t total_pages; /* total number of 4kB pages */ + + /* Virtual memory region information */ + const struct sys_mm_drv_region *virtual_region; + struct sys_mem_blocks vpage_blocks; + + /* allocation elements to track page id to allocation size */ + uint32_t num_elems_in_use; /* number of allocated elements in use*/ + struct valloc_elem velems[VPAGE_MAX_ALLOCS]; +}; + +/* uncache persistent across all cores */ +static struct vpage_context page_context; +static sys_bitarray_t bitmap; + +/* singleton across all cores */ +static int vpage_init_done; + +/** + * @brief Allocate and map virtual memory pages + * + * Allocates memory pages from the virtual page allocator. + * Maps physical memory pages to the virtual region as needed. + * + * @param pages Number of 4kB pages to allocate. + * @param ptr Pointer to store the address of allocated pages. + * @retval 0 if successful. + */ +static int vpages_alloc_and_map(uint32_t pages, void **ptr) +{ + void *vaddr; + int ret; + + /* check for valid pages and ptr */ + if (!pages || !ptr) + return -EINVAL; + + *ptr = NULL; + + /* quick check for enough free pages */ + if (page_context.free_pages < pages) { + LOG_ERR("error: not enough free pages %d for requested pages %d", + page_context.free_pages, pages); + return -ENOMEM; + } + + /* check for allocation elements */ + if (page_context.num_elems_in_use >= VPAGE_MAX_ALLOCS) { + LOG_ERR("error: max allocation elements reached"); + return -ENOMEM; + } + + /* allocate virtual continuous blocks */ + ret = sys_mem_blocks_alloc_contiguous(&page_context.vpage_blocks, pages, &vaddr); + if (ret < 0) { + LOG_ERR("error: failed to allocate %d continuous virtual pages, free %d", + pages, page_context.free_pages); + return ret; + } + + /* map the virtual blocks in virtual region to free physical blocks */ + ret = sys_mm_drv_map_region_safe(page_context.virtual_region, vaddr, + 0, pages * CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW); + if (ret < 0) { + LOG_ERR("error: failed to map virtual region %p to physical region %p, error %d", + vaddr, page_context.virtual_region->addr, ret); + sys_mem_blocks_free(&page_context.vpage_blocks, pages, &vaddr); + return ret; + } + + /* success update the free pages */ + page_context.free_pages -= pages; + + /* store the size and virtual page number in first free alloc element, + * we have already checked for a free element before the mapping. + */ + for (int i = 0; i < VPAGE_MAX_ALLOCS; i++) { + if (page_context.velems[i].pages == 0) { + page_context.velems[i].pages = pages; + page_context.velems[i].vpage = + (POINTER_TO_UINT(vaddr) - + POINTER_TO_UINT(page_context.vpage_blocks.buffer)) / + CONFIG_MM_DRV_PAGE_SIZE; + page_context.num_elems_in_use++; + break; + } + } + + /* return the virtual address */ + *ptr = vaddr; + return ret; +} + +/** + * @brief Allocate virtual memory pages + * + * Allocates virtual memory pages from the virtual page allocator. + * + * @param pages Number of 4kB pages to allocate. + * @retval NULL on allocation failure. + */ +void *alloc_vpages(uint32_t pages) +{ + void *ptr = NULL; + int err; + + k_mutex_lock(&page_context.lock, K_FOREVER); + err = vpages_alloc_and_map(pages, &ptr); + k_mutex_unlock(&page_context.lock); + if (err < 0) { + LOG_ERR("vpage_alloc failed %d for %d pages, total %d free %d", + err, pages, page_context.total_pages, page_context.free_pages); + } + LOG_INF("vpage_alloc ptr %p pages %d free %d/%d", ptr, pages, page_context.free_pages, + page_context.total_pages); + return ptr; +} + +/** + * @brief Free and unmap virtual memory pages + * + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr Pointer to the memory pages to free. + * @retval 0 if successful. + * @retval -EINVAL if ptr is invalid. + */ +static int vpages_free_and_unmap(uintptr_t *ptr) +{ + int pages = 0; + int ret; + + /* check for valid ptr which must be page aligned */ + if (IS_ALIGNED(ptr, CONFIG_MM_DRV_PAGE_SIZE) == 0) { + LOG_ERR("error: invalid non aligned page pointer %p", ptr); + return -EINVAL; + } + + /* find the allocation element */ + for (int i = 0; i < VPAGE_MAX_ALLOCS; i++) { + if (page_context.velems[i].pages > 0 && + page_context.velems[i].vpage == + (POINTER_TO_UINT(ptr) - POINTER_TO_UINT(page_context.vpage_blocks.buffer)) / + CONFIG_MM_DRV_PAGE_SIZE) { + + pages = page_context.velems[i].pages; + + LOG_DBG("found allocation element %d pages %d vpage %d for ptr %p", + i, page_context.velems[i].pages, + page_context.velems[i].vpage, ptr); + + /* clear the element */ + page_context.velems[i].pages = 0; + page_context.velems[i].vpage = 0; + page_context.num_elems_in_use--; + break; + } + } + + /* check we found allocation element */ + if (pages == 0) { + LOG_ERR("error: invalid page pointer %p not found", ptr); + return -EINVAL; + } + + /* unmap the pages from virtual region */ + ret = sys_mm_drv_unmap_region((void *)ptr, pages * CONFIG_MM_DRV_PAGE_SIZE); + if (ret < 0) { + LOG_ERR("error: failed to unmap virtual region %p pages %d, error %d", + ptr, pages, ret); + return ret; + } + + /* free physical blocks */ + ret = sys_mem_blocks_free_contiguous(&page_context.vpage_blocks, ptr, pages); + if (ret < 0) { + LOG_ERR("error: failed to free %d continuous virtual page blocks at %p, error %d", + pages, ptr, ret); + return ret; + } + + /* success update the free pages */ + page_context.free_pages += pages; + return ret; +} + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr + */ +void free_vpages(void *ptr) +{ + int err; + + k_mutex_lock(&page_context.lock, K_FOREVER); + err = vpages_free_and_unmap((uintptr_t *)ptr); + k_mutex_unlock(&page_context.lock); + + /* should never fail */ + assert(!err); + + LOG_INF("vptr %p free/total pages %d/%d", ptr, page_context.free_pages, + page_context.total_pages); +} + +/** + * @brief Initialize virtual page allocator + * + * Initializes a virtual page allocator that manages a virtual memory region + * using a page table and block structures. + * + * @retval 0 if successful. + * @retval -ENOMEM on creation failure. + */ +static int init_vpages(void) +{ + const struct sys_mm_drv_region *virtual_memory_regions; + const struct sys_mm_drv_region *region; + uint32_t *bundles = NULL; + size_t block_count, bitmap_num_bundles; + int ret; + + /* Check if already initialized */ + if (vpage_init_done) + return 0; + + /* create the virtual memory region and add it to the system */ + ret = adsp_add_virtual_memory_region(adsp_mm_get_unused_l2_start_aligned(), + CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE, + VIRTUAL_REGION_SHARED_HEAP_ATTR); + if (ret) + return ret; + + memset(&page_context, 0, sizeof(page_context)); + k_mutex_init(&page_context.lock); + + /* now find the virtual region in all memory regions */ + virtual_memory_regions = sys_mm_drv_query_memory_regions(); + SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, region) { + if (region->attr == VIRTUAL_REGION_SHARED_HEAP_ATTR) { + page_context.virtual_region = region; + break; + } + } + sys_mm_drv_query_memory_regions_free(virtual_memory_regions); + + /* check for a valid region */ + if (!page_context.virtual_region) { + LOG_ERR("error: no valid virtual region found"); + return -EINVAL; + } + + block_count = region->size / CONFIG_MM_DRV_PAGE_SIZE; + if (block_count == 0) { + LOG_ERR("error: virtual region too small %d", region->size); + return -ENOMEM; + } + page_context.total_pages = block_count; + page_context.free_pages = block_count; + page_context.num_elems_in_use = 0; + + /* bundles are uint32_t of bits */ + bitmap_num_bundles = SOF_DIV_ROUND_UP(block_count, 32); + + /* allocate memory for bitmap bundles */ + bundles = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + bitmap_num_bundles * sizeof(uint32_t)); + if (!bundles) { + LOG_ERR("error: virtual region bitmap alloc failed"); + return -ENOMEM; + } + + /* Fill allocators data based on config and virtual region data */ + page_context.vpage_blocks.info.num_blocks = block_count; + page_context.vpage_blocks.info.blk_sz_shift = ilog2(CONFIG_MM_DRV_PAGE_SIZE); + /* buffer is the start of the virtual memory region */ + page_context.vpage_blocks.buffer = (uint8_t *)page_context.virtual_region->addr; + + /* initialize bitmap */ + bitmap.num_bits = block_count; + bitmap.num_bundles = bitmap_num_bundles; + bitmap.bundles = bundles; + page_context.vpage_blocks.bitmap = &bitmap; + + LOG_INF("vpage_init region %p size 0x%x pages %d", + (void *)page_context.virtual_region->addr, + (int)page_context.virtual_region->size, block_count); + + vpage_init_done = 1; + return 0; +} + +SYS_INIT(init_vpages, POST_KERNEL, 1); + From 9e93e120f78713b3e31ddf7f6feb5956a2aeec39 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 23 Oct 2025 13:39:49 +0100 Subject: [PATCH 2/2] vregion: Add support for per pipeline/module virtual regions Add support for per pipeline and per module virtual memory regions. The intention is to provide a single virtual memory region per pipeline or per DP module that can simplify module/pipeline memory management. The region takes advantage of the way pipeline and modules are constructed, destroyed and used during their lifetimes. 1) memory tracking - 1 pointer/size per pipeline or DP module. 2) memory read/write/execute permissions 3) memory sharing across cores and domains. 4) cache utilization. Modules and pipelines will allocate from their region only and this will be abstracted via the allocation APIs. Signed-off-by: Liam Girdwood --- zephyr/CMakeLists.txt | 16 +- zephyr/Kconfig | 15 +- zephyr/include/sof/lib/vregion.h | 106 ++++++++ zephyr/lib/vregion.c | 434 +++++++++++++++++++++++++++++++ 4 files changed, 566 insertions(+), 5 deletions(-) create mode 100644 zephyr/include/sof/lib/vregion.h create mode 100644 zephyr/lib/vregion.c diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 61ed49a3d15f..05ad5670931f 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -287,10 +287,18 @@ if (CONFIG_SOC_SERIES_INTEL_ADSP_ACE) ${SOF_PLATFORM_PATH}/novalake/lib/clk.c ) - # Sources for virtual heap management - zephyr_library_sources( - lib/regions_mm.c - ) + # Virtual memory support is required and can be enabled with + # either VMH or Virtual pages and regions. + if (CONFIG_SOF_VREGIONS) + zephyr_library_sources( + lib/vpages.c + lib/vregion.c + ) + else() + zephyr_library_sources( + lib/regions_mm.c + ) + endif() zephyr_library_sources_ifdef(CONFIG_CAVS_LPS ${SOF_PLATFORM_PATH}/intel/ace/lps_wait.c diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 3d7e8b3943d7..5affb4d5e716 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -148,6 +148,19 @@ config SOF_VPAGE_MAX_ALLOCS region. Increasing this number allows for more simultaneous page allocations, but also increases the memory overhead for tracking these allocations. +config SOF_VREGIONS + bool "Enable virtual memory regions" + default n + depends on ACE + help + Enable the virtual regions memory allocator for pipeline resource management. + This provides a way to manage memory resources for audio pipelines, + including + 1) multiple pipeline static lifetime allocations. + 2) runtime pipeline allocations. + 3) pipeline shared memory allocations. + 4) module text allocation. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" help @@ -242,7 +255,7 @@ config SOF_ZEPHYR_NO_SOF_CLOCK config VIRTUAL_HEAP bool "Use virtual memory heap to allocate a buffers" - default y if ACE + default n depends on ACE help Enabling this option will use the virtual memory heap allocator to allocate buffers. diff --git a/zephyr/include/sof/lib/vregion.h b/zephyr/include/sof/lib/vregion.h new file mode 100644 index 000000000000..95baf0210774 --- /dev/null +++ b/zephyr/include/sof/lib/vregion.h @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Pre Allocated Contiguous Virtual Region */ +#ifndef __SOF_LIB_VREGION_H__ +#define __SOF_LIB_VREGION_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct vregion; + +/** + * @brief Create a new virtual region instance. + * + * Create a new virtual region instance with specified static, dynamic, and shared static sizes + * plus an optional read-only text partition and optional shared static partition. + * Total size is the sum of static, dynamic, shared static, and text sizes. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @param[in] lifetime_shared_size Size of the virtual region shared lifetime partition. + * @param[in] interim_shared_size Size of the virtual region shared interim partition. + * @param[in] text_size Size of the optional read-only text partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size, + size_t lifetime_shared_size, size_t interim_shared_size, + size_t text_size); + +/** + * @brief Destroy a virtual region instance. + * + * Free all associated resources and deallocate the virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr); + +/** + * @brief Memory types for virtual region allocations. + * Used to specify the type of memory allocation within a virtual region. + * + * @note + * - interim: allocation that can be freed i.e. get/set large config, kcontrols. + * - lifetime: allocation that cannot be freed i.e. init data, pipeline data. + * - shared: allocation that can be shared between multiple virtual regions and + * - with different cores. + */ +enum vregion_mem_type { + VREGION_MEM_TYPE_INTERIM, /* interim allocation that can be freed */ + VREGION_MEM_TYPE_LIFETIME, /* lifetime allocation */ + VREGION_MEM_TYPE_INTERIM_SHARED, /* shared interim allocation */ + VREGION_MEM_TYPE_LIFETIME_SHARED /* shared lifetime allocation */ +}; + +/** + * @brief Allocate memory from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (static, dynamic, or shared static). + * @param[in] size Size of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size); + +/** + * @brief Allocate aligned memory from the specified virtual region. + * + * Allocate aligned memory from the specified virtual region based on the memory type. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (static, dynamic, or shared static). + * @param[in] size Size of memory to allocate in bytes. + * @param[in] alignment Alignment of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment); + +/** + * @brief Free memory allocated from the specified virtual region. + * + * Free memory previously allocated from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr); + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VREGION_H__ */ diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c new file mode 100644 index 000000000000..3651eda785b3 --- /dev/null +++ b/zephyr/lib/vregion.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vregion, CONFIG_SOF_LOG_LEVEL); + +/* + * Pre Allocated Contiguous Virtual Memory Region Allocator + * + * This allocator manages a pre-allocated virtual memory region that uses the + * virtual page allocator to allocate and free memory pages. + * + * It is designed for use cases where a contiguous virtual memory region + * is required, such as for batched allocation of audio pipelines and modules. + * + * New pipelines will create a new virtual region and will specify the size of the region + * which can be divided into multiple areas for different allocation lifetimes, permissions + * and sharing requirements. + * + * Advantages: + * + * 1) Contiguous virtual memory region for easier management and tracking of + * pipeline & DP module memory. i.e. we just need to track the vregion pointer. + * 2) Easier management of memory protection and sharing between different cores + * and domains by partitioning the virtual region into different areas with + * specific permissions and sharing requirements. + * 3) Reduced fragmentation and better cache utilization by using a simple linear + * allocator for lifetime objects. + * + * Note: Software must pass in the size of the region areas at pipeline creation time. + */ + +/** + * @brief virtual region memory region structure. + * + * This structure represents a virtual memory region, which includes + * information about the base address, size, and allocation status + * of the region. + * + * The virtual region memory region can be partitioned into five main areas on + * page-aligned boundaries (some are optional), listed here from base to top: + * + * 1. Text Region (optional): A read-only and executable region that can be used + * to store code or constant data. This region is optional and only present + * if the virtual region is created with a text size. It is page aligned and located + * at the start of the virtual region. Main use case would be DP module text. + * + * 2. Interim Heap: A interim memory area used for multiple temporary allocations + * and frees over the lifetime of the audio processing pipeline. e.g. for + * module kcontrol derived allocations/frees. + * + * 3. Shared Interim Heap (optional): A interim memory area used for multiple temporary + * allocations/frees that can be shared between multiple cores or memory domains. e.g + * shared buffers between different cores or domains. + * + * 4. Lifetime Allocator: A simple incrementing allocator used for long-term static + * allocations that persist for the lifetime of the audio processing pipeline. This + * allocator compresses allocations for better cache utilization. + * + * 5. Shared Lifetime Allocator (optional): A simple incrementing allocator used for long + * term static allocations that can be shared between multiple cores or domains. This + * allocator aligns allocations to cache line boundaries to ensure cache coherency. + * + * * TODO: Pipeline/module reset() could reset the dynamic heap. + */ + + /* linear heap used for lifetime allocations */ +struct vlinear_heap { + uint8_t *base; /* base address of linear allocator */ + uint8_t *ptr; /* current alloc pointer */ + size_t size; /* size of linear allocator in bytes */ + size_t used; /* used bytes in linear allocator */ + int free_count; /* number of frees - tuning only */ +}; + +/* zephyr k_heap for interim allocations. TODO: make lockless for improved performance */ +struct zephyr_heap { + struct k_heap heap; + uint8_t *base; /* base address of zephyr heap allocator */ + size_t size; /* size of heap in bytes */ +}; + +/* Main vregion context, see above intro for more details. + * TODO: Add support to flag which heaps should have their contexts saved and restored. + */ +struct vregion { + /* region context */ + uint8_t *base; /* base address of entire region */ + size_t size; /* size of whole region in bytes */ + size_t pages; /* size of whole region in pages */ + + /* optional text region - RO and Executable */ + struct vlinear_heap text; /* text linear heap */ + + /* interim heap */ + struct zephyr_heap interim; /* interim heap */ + + /* interim shared */ + struct zephyr_heap interim_shared; /* shared interim heap */ + + /* lifetime heap */ + struct vlinear_heap lifetime; /* lifetime linear heap */ + + /* optional shared static buffer heap */ + struct vlinear_heap lifetime_shared; /* shared lifetime linear heap */ +}; + +/** + * @brief Create a new virtual region instance with shared pages. + * + * Create a new VIRTUAL REGION instance with specified static, dynamic, and shared static sizes. + * Total size is the sum of static, dynamic, and shared static sizes. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @param[in] lifetime_shared_size Size of the virtual region shared lifetime partition. + * @param[in] interim_shared_size Size of the virtual region shared interim partition. + * @param[in] text_size Size of the optional read-only text partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size, + size_t lifetime_shared_size, size_t interim_shared_size, + size_t text_size) +{ + struct vregion *vr; + uint32_t pages; + size_t total_size; + uint8_t *vregion_base; + + if (!lifetime_size || !interim_size) { + LOG_ERR("error: invalid vregion lifetime size %d or interim size %d", + lifetime_size, interim_size); + return NULL; + } + + /* + * Align up lifetime sizes and interim sizes to nearest page, the + * vregion structure is stored in lifetime area so account for its size too. + */ + lifetime_size += sizeof(*vr); + lifetime_size = ALIGN_UP(lifetime_size, CONFIG_MM_DRV_PAGE_SIZE); + interim_size = ALIGN_UP(interim_size, CONFIG_MM_DRV_PAGE_SIZE); + lifetime_shared_size = ALIGN_UP(lifetime_shared_size, CONFIG_MM_DRV_PAGE_SIZE); + interim_shared_size = ALIGN_UP(interim_shared_size, CONFIG_MM_DRV_PAGE_SIZE); + text_size = ALIGN_UP(text_size, CONFIG_MM_DRV_PAGE_SIZE); + total_size = lifetime_size + interim_size + + lifetime_shared_size + interim_shared_size + text_size; + + /* allocate pages for vregion */ + pages = total_size / CONFIG_MM_DRV_PAGE_SIZE; + vregion_base = alloc_vpages(pages); + if (!vregion_base) + return NULL; + + /* init vregion - place it at the start of the lifetime region */ + vr = (struct vregion *)(vregion_base + text_size + interim_size); + vr->base = vregion_base; + vr->size = total_size; + vr->pages = pages; + + /* set partition sizes */ + vr->interim.size = interim_size; + vr->interim_shared.size = interim_shared_size; + vr->lifetime.size = lifetime_size; + vr->lifetime_shared.size = lifetime_shared_size; + vr->text.size = text_size; + + /* set base addresses for partitions */ + vr->text.base = vr->base; + vr->interim.base = vr->text.base + text_size; + vr->lifetime.base = vr->interim.base + interim_size; + vr->lifetime_shared.base = vr->lifetime.base + lifetime_size; + vr->interim_shared.base = vr->lifetime_shared.base + lifetime_shared_size; + + /* set alloc ptr addresses for lifetime linear partitions */ + vr->text.ptr = vr->text.base; + vr->lifetime.ptr = vr->lifetime.base + sizeof(*vr); /* skip vregion struct */ + vr->lifetime.used = sizeof(*vr); + vr->lifetime_shared.ptr = vr->lifetime_shared.base; + + /* init interim heaps */ + k_heap_init(&vr->interim.heap, vr->interim.base, interim_size); + if (interim_shared_size) { + k_heap_init(&vr->interim_shared.heap, vr->interim_shared.base, + interim_shared_size); + } + + /* log the new vregion */ + LOG_INF("new at base %p size 0x%x pages %d struct embedded at %p", + (void *)vr->base, total_size, pages, (void *)vr); + LOG_INF(" interim size 0x%x at %p", interim_size, (void *)vr->interim.base); + LOG_INF(" lifetime size 0x%x at %p", lifetime_size, (void *)vr->lifetime.base); + if (interim_shared_size) + LOG_INF(" interim shared size 0x%x at %p", interim_shared_size, + (void *)vr->interim_shared.base); + if (lifetime_shared_size) + LOG_INF(" lifetime shared size 0x%x at %p", lifetime_shared_size, + (void *)vr->lifetime_shared.base); + if (text_size) + LOG_INF(" text size 0x%x at %p", text_size, (void *)vr->text.base); + + return vr; +} + +/** + * @brief Destroy a virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr) +{ + if (!vr) + return; + + /* log the vregion being destroyed */ + LOG_INF("destroy %p size 0x%x pages %d", + (void *)vr->base, vr->size, vr->pages); + LOG_INF(" lifetime used %d free count %d", + vr->lifetime.used, vr->lifetime.free_count); + if (vr->lifetime_shared.size) + LOG_INF(" lifetime shared used %d free count %d", + vr->lifetime_shared.used, vr->lifetime_shared.free_count); + free_vpages(vr->base); +} + +/** + * @brief Allocate memory with alignment from the virtual region dynamic heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *interim_alloc(struct zephyr_heap *heap, + size_t size, size_t align) +{ + void *ptr; + + ptr = k_heap_aligned_alloc(&heap->heap, size, align, K_FOREVER); + if (!ptr) { + LOG_ERR("error: interim alloc failed for %d bytes align %d", + size, align); + return NULL; + } + + return ptr; +} + +/** + * @brief Free memory from the virtual region interim heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void interim_free(struct zephyr_heap *heap, void *ptr) +{ + k_heap_free(&heap->heap, ptr); +} + +/** + * @brief Allocate memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *lifetime_alloc(struct vlinear_heap *heap, + size_t size, size_t align) +{ + void *ptr; + uint8_t *aligned_ptr; + size_t heap_obj_size; + + /* align heap pointer to alignment requested */ + aligned_ptr = UINT_TO_POINTER(ALIGN_UP(POINTER_TO_UINT(heap->ptr), align)); + + /* also align up size to D$ bytes if asked - allocation head and tail aligned */ + if (align == CONFIG_DCACHE_LINE_SIZE) + size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); + + /* calculate new heap object size for object and alignments */ + heap_obj_size = aligned_ptr - heap->ptr + size; + + /* check we have enough lifetime space left */ + if (heap_obj_size + heap->used > heap->size) { + LOG_ERR("error: lifetime alloc failed for object %d heap %d bytes free %d", + size, heap_obj_size, heap->size - heap->used); + return NULL; + } + + /* allocate memory */ + ptr = aligned_ptr; + heap->ptr += heap_obj_size; + heap->used += heap_obj_size; + + return ptr; +} + +/** + * @brief Free memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void lifetime_free(struct vlinear_heap *heap, void *ptr) +{ + /* simple free, just increment free count, this is for tuning only */ + heap->free_count++; + + LOG_DBG("lifetime free %p count %d", ptr, heap->free_count); +} + +/** + * @brief Free memory from the virtual region. + * + * @param vr Pointer to the virtual region instance. + * @param ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr) +{ + if (!vr || !ptr) + return; + + /* check if pointer is in interim heap */ + if (ptr >= (void *)vr->interim.base && + ptr < (void *)(vr->interim.base + vr->interim.size)) { + interim_free(&vr->interim, ptr); + return; + } + + /* check if pointer is in interim shared heap */ + if (vr->interim_shared.size && + ptr >= (void *)vr->interim_shared.base && + ptr < (void *)(vr->interim_shared.base + vr->interim_shared.size)) { + interim_free(&vr->interim_shared, ptr); + return; + } + + /* check if pointer is in lifetime heap */ + if (ptr >= (void *)vr->lifetime.base && + ptr < (void *)(vr->lifetime.base + vr->lifetime.size)) { + lifetime_free(&vr->lifetime, ptr); + return; + } + + /* check if pointer is in lifetime shared heap */ + if (vr->lifetime_shared.size && + ptr >= (void *)vr->lifetime_shared.base && + ptr < (void *)(vr->lifetime_shared.base + vr->lifetime_shared.size)) { + lifetime_free(&vr->lifetime_shared, ptr); + return; + } + + LOG_ERR("error: vregion free invalid pointer %p", ptr); +} + +/** + * @brief Allocate memory type from the virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @param[in] alignment Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment) +{ + if (!vr || !size) + return NULL; + + if (!alignment) + alignment = 4; /* default align 4 bytes */ + + switch (type) { + case VREGION_MEM_TYPE_INTERIM: + return interim_alloc(&vr->interim, size, alignment); + case VREGION_MEM_TYPE_LIFETIME: + return lifetime_alloc(&vr->lifetime, size, alignment); + case VREGION_MEM_TYPE_INTERIM_SHARED: + return interim_alloc(&vr->interim_shared, size, alignment); + case VREGION_MEM_TYPE_LIFETIME_SHARED: + return lifetime_alloc(&vr->lifetime_shared, size, + MAX(alignment, CONFIG_DCACHE_LINE_SIZE)); + default: + LOG_ERR("error: invalid memory type %d", type); + return NULL; + } +} + +/** + * @brief Allocate memory from the virtual region. + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) +{ + return vregion_alloc_align(vr, type, size, 0); +} + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr) +{ + if (!vr) + return; + + LOG_INF("base %p size 0x%x pages %d", + (void *)vr->base, vr->size, vr->pages); + LOG_INF("lifetime used 0x%x free count %d", + vr->lifetime.used, vr->lifetime.free_count); + LOG_INF("lifetime shared used 0x%x free count %d", + vr->lifetime_shared.used, vr->lifetime_shared.free_count); +} +EXPORT_SYMBOL(vregion_info);