diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 61ed49a3d15f..05ad5670931f 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -287,10 +287,18 @@ if (CONFIG_SOC_SERIES_INTEL_ADSP_ACE) ${SOF_PLATFORM_PATH}/novalake/lib/clk.c ) - # Sources for virtual heap management - zephyr_library_sources( - lib/regions_mm.c - ) + # Virtual memory support is required and can be enabled with + # either VMH or Virtual pages and regions. + if (CONFIG_SOF_VREGIONS) + zephyr_library_sources( + lib/vpages.c + lib/vregion.c + ) + else() + zephyr_library_sources( + lib/regions_mm.c + ) + endif() zephyr_library_sources_ifdef(CONFIG_CAVS_LPS ${SOF_PLATFORM_PATH}/intel/ace/lps_wait.c diff --git a/zephyr/Kconfig b/zephyr/Kconfig index a642ee52471c..5affb4d5e716 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -138,6 +138,29 @@ config SOF_USERSPACE_APPLICATION Not manually settable. This is effectively a shortcut to replace numerous checks for (CONFIG_USERSPACE && !CONFIG_SOF_USERSPACE_PROXY) +config SOF_VPAGE_MAX_ALLOCS + int "Number of virtual memory page allocation elements" + default 128 + help + This setting defines the maximum number of virtual memory page allocation + elements that can be tracked. Each allocation element represents a + contiguous block of virtual memory allocated from the virtual memory + region. Increasing this number allows for more simultaneous page allocations, + but also increases the memory overhead for tracking these allocations. + +config SOF_VREGIONS + bool "Enable virtual memory regions" + default n + depends on ACE + help + Enable the virtual regions memory allocator for pipeline resource management. + This provides a way to manage memory resources for audio pipelines, + including + 1) multiple pipeline static lifetime allocations. + 2) runtime pipeline allocations. + 3) pipeline shared memory allocations. + 4) module text allocation. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" help @@ -232,7 +255,7 @@ config SOF_ZEPHYR_NO_SOF_CLOCK config VIRTUAL_HEAP bool "Use virtual memory heap to allocate a buffers" - default y if ACE + default n depends on ACE help Enabling this option will use the virtual memory heap allocator to allocate buffers. diff --git a/zephyr/include/sof/lib/vpages.h b/zephyr/include/sof/lib/vpages.h new file mode 100644 index 000000000000..a360c6bf3a8a --- /dev/null +++ b/zephyr/include/sof/lib/vpages.h @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Virtual Page Allocator API */ +#ifndef __SOF_LIB_VPAGE_H__ +#define __SOF_LIB_VPAGE_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Allocate virtual pages + * Allocates a specified number of contiguous virtual memory pages by mapping + * physical pages. + * + * @param[in] pages Number of 4kB pages to allocate. + * + * @return Pointer to the allocated virtual memory region, or NULL on failure. + */ +void *alloc_vpages(uint32_t pages); + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param[in] ptr Pointer to the memory pages to free. + */ +void free_vpages(void *ptr); + +/** + * @brief Initialize virtual page allocator + * + * Initializes a virtual page allocator that manages a virtual memory region + * using a page table and block structures. + * + * @retval 0 if successful. + * @retval -ENOMEM on creation failure. + */ +int init_vpages(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VPAGE_H__ */ diff --git a/zephyr/include/sof/lib/vregion.h b/zephyr/include/sof/lib/vregion.h new file mode 100644 index 000000000000..95baf0210774 --- /dev/null +++ b/zephyr/include/sof/lib/vregion.h @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Pre Allocated Contiguous Virtual Region */ +#ifndef __SOF_LIB_VREGION_H__ +#define __SOF_LIB_VREGION_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct vregion; + +/** + * @brief Create a new virtual region instance. + * + * Create a new virtual region instance with specified static, dynamic, and shared static sizes + * plus an optional read-only text partition and optional shared static partition. + * Total size is the sum of static, dynamic, shared static, and text sizes. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @param[in] lifetime_shared_size Size of the virtual region shared lifetime partition. + * @param[in] interim_shared_size Size of the virtual region shared interim partition. + * @param[in] text_size Size of the optional read-only text partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size, + size_t lifetime_shared_size, size_t interim_shared_size, + size_t text_size); + +/** + * @brief Destroy a virtual region instance. + * + * Free all associated resources and deallocate the virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr); + +/** + * @brief Memory types for virtual region allocations. + * Used to specify the type of memory allocation within a virtual region. + * + * @note + * - interim: allocation that can be freed i.e. get/set large config, kcontrols. + * - lifetime: allocation that cannot be freed i.e. init data, pipeline data. + * - shared: allocation that can be shared between multiple virtual regions and + * - with different cores. + */ +enum vregion_mem_type { + VREGION_MEM_TYPE_INTERIM, /* interim allocation that can be freed */ + VREGION_MEM_TYPE_LIFETIME, /* lifetime allocation */ + VREGION_MEM_TYPE_INTERIM_SHARED, /* shared interim allocation */ + VREGION_MEM_TYPE_LIFETIME_SHARED /* shared lifetime allocation */ +}; + +/** + * @brief Allocate memory from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (static, dynamic, or shared static). + * @param[in] size Size of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size); + +/** + * @brief Allocate aligned memory from the specified virtual region. + * + * Allocate aligned memory from the specified virtual region based on the memory type. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (static, dynamic, or shared static). + * @param[in] size Size of memory to allocate in bytes. + * @param[in] alignment Alignment of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment); + +/** + * @brief Free memory allocated from the specified virtual region. + * + * Free memory previously allocated from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr); + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VREGION_H__ */ diff --git a/zephyr/lib/vpages.c b/zephyr/lib/vpages.c new file mode 100644 index 000000000000..fcd7eef3c987 --- /dev/null +++ b/zephyr/lib/vpages.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vpage, CONFIG_SOF_LOG_LEVEL); + +/* Simple Page Allocator. + * + * This allocator manages the allocation and deallocation of virtual memory pages from + * a predefined virtual memory region which is larger than the physical memory region. + * + * Both memory regions are divided into 4kB pages that are represented as blocks in a + * bitmap using the zephyr sys_mem_blocks API. The virtual block map tracks the allocation + * of virtual memory pages while the physical block map in the Zephyr MM driver tracks + * the allocation of physical memory pages. + */ + +/* max number of allocation elements */ +#define VPAGE_MAX_ALLOCS CONFIG_SOF_VPAGE_MAX_ALLOCS + +/* + * Virtual memory allocation element - tracks allocated virtual page id and size + */ +struct valloc_elem { + uint16_t pages; /* number of 4kB pages allocated in contiguous block */ + uint16_t vpage; /* virtual page number from start of region */ +} __packed; + +/* + * Virtual page table structure + * + * This structure holds all information about virtual memory pages + * including the number of free and total pages, the virtual memory + * region, the block allocator for virtual pages and the allocation + * elements. + */ +struct vpage_context { + struct k_mutex lock; + uint32_t free_pages; /* number of free 4kB pages */ + uint32_t total_pages; /* total number of 4kB pages */ + + /* Virtual memory region information */ + const struct sys_mm_drv_region *virtual_region; + struct sys_mem_blocks vpage_blocks; + + /* allocation elements to track page id to allocation size */ + uint32_t num_elems_in_use; /* number of allocated elements in use*/ + struct valloc_elem velems[VPAGE_MAX_ALLOCS]; +}; + +/* uncache persistent across all cores */ +static struct vpage_context page_context; +static sys_bitarray_t bitmap; + +/* singleton across all cores */ +static int vpage_init_done; + +/** + * @brief Allocate and map virtual memory pages + * + * Allocates memory pages from the virtual page allocator. + * Maps physical memory pages to the virtual region as needed. + * + * @param pages Number of 4kB pages to allocate. + * @param ptr Pointer to store the address of allocated pages. + * @retval 0 if successful. + */ +static int vpages_alloc_and_map(uint32_t pages, void **ptr) +{ + void *vaddr; + int ret; + + /* check for valid pages and ptr */ + if (!pages || !ptr) + return -EINVAL; + + *ptr = NULL; + + /* quick check for enough free pages */ + if (page_context.free_pages < pages) { + LOG_ERR("error: not enough free pages %d for requested pages %d", + page_context.free_pages, pages); + return -ENOMEM; + } + + /* check for allocation elements */ + if (page_context.num_elems_in_use >= VPAGE_MAX_ALLOCS) { + LOG_ERR("error: max allocation elements reached"); + return -ENOMEM; + } + + /* allocate virtual continuous blocks */ + ret = sys_mem_blocks_alloc_contiguous(&page_context.vpage_blocks, pages, &vaddr); + if (ret < 0) { + LOG_ERR("error: failed to allocate %d continuous virtual pages, free %d", + pages, page_context.free_pages); + return ret; + } + + /* map the virtual blocks in virtual region to free physical blocks */ + ret = sys_mm_drv_map_region_safe(page_context.virtual_region, vaddr, + 0, pages * CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW); + if (ret < 0) { + LOG_ERR("error: failed to map virtual region %p to physical region %p, error %d", + vaddr, page_context.virtual_region->addr, ret); + sys_mem_blocks_free(&page_context.vpage_blocks, pages, &vaddr); + return ret; + } + + /* success update the free pages */ + page_context.free_pages -= pages; + + /* store the size and virtual page number in first free alloc element, + * we have already checked for a free element before the mapping. + */ + for (int i = 0; i < VPAGE_MAX_ALLOCS; i++) { + if (page_context.velems[i].pages == 0) { + page_context.velems[i].pages = pages; + page_context.velems[i].vpage = + (POINTER_TO_UINT(vaddr) - + POINTER_TO_UINT(page_context.vpage_blocks.buffer)) / + CONFIG_MM_DRV_PAGE_SIZE; + page_context.num_elems_in_use++; + break; + } + } + + /* return the virtual address */ + *ptr = vaddr; + return ret; +} + +/** + * @brief Allocate virtual memory pages + * + * Allocates virtual memory pages from the virtual page allocator. + * + * @param pages Number of 4kB pages to allocate. + * @retval NULL on allocation failure. + */ +void *alloc_vpages(uint32_t pages) +{ + void *ptr = NULL; + int err; + + k_mutex_lock(&page_context.lock, K_FOREVER); + err = vpages_alloc_and_map(pages, &ptr); + k_mutex_unlock(&page_context.lock); + if (err < 0) { + LOG_ERR("vpage_alloc failed %d for %d pages, total %d free %d", + err, pages, page_context.total_pages, page_context.free_pages); + } + LOG_INF("vpage_alloc ptr %p pages %d free %d/%d", ptr, pages, page_context.free_pages, + page_context.total_pages); + return ptr; +} + +/** + * @brief Free and unmap virtual memory pages + * + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr Pointer to the memory pages to free. + * @retval 0 if successful. + * @retval -EINVAL if ptr is invalid. + */ +static int vpages_free_and_unmap(uintptr_t *ptr) +{ + int pages = 0; + int ret; + + /* check for valid ptr which must be page aligned */ + if (IS_ALIGNED(ptr, CONFIG_MM_DRV_PAGE_SIZE) == 0) { + LOG_ERR("error: invalid non aligned page pointer %p", ptr); + return -EINVAL; + } + + /* find the allocation element */ + for (int i = 0; i < VPAGE_MAX_ALLOCS; i++) { + if (page_context.velems[i].pages > 0 && + page_context.velems[i].vpage == + (POINTER_TO_UINT(ptr) - POINTER_TO_UINT(page_context.vpage_blocks.buffer)) / + CONFIG_MM_DRV_PAGE_SIZE) { + + pages = page_context.velems[i].pages; + + LOG_DBG("found allocation element %d pages %d vpage %d for ptr %p", + i, page_context.velems[i].pages, + page_context.velems[i].vpage, ptr); + + /* clear the element */ + page_context.velems[i].pages = 0; + page_context.velems[i].vpage = 0; + page_context.num_elems_in_use--; + break; + } + } + + /* check we found allocation element */ + if (pages == 0) { + LOG_ERR("error: invalid page pointer %p not found", ptr); + return -EINVAL; + } + + /* unmap the pages from virtual region */ + ret = sys_mm_drv_unmap_region((void *)ptr, pages * CONFIG_MM_DRV_PAGE_SIZE); + if (ret < 0) { + LOG_ERR("error: failed to unmap virtual region %p pages %d, error %d", + ptr, pages, ret); + return ret; + } + + /* free physical blocks */ + ret = sys_mem_blocks_free_contiguous(&page_context.vpage_blocks, ptr, pages); + if (ret < 0) { + LOG_ERR("error: failed to free %d continuous virtual page blocks at %p, error %d", + pages, ptr, ret); + return ret; + } + + /* success update the free pages */ + page_context.free_pages += pages; + return ret; +} + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr + */ +void free_vpages(void *ptr) +{ + int err; + + k_mutex_lock(&page_context.lock, K_FOREVER); + err = vpages_free_and_unmap((uintptr_t *)ptr); + k_mutex_unlock(&page_context.lock); + + /* should never fail */ + assert(!err); + + LOG_INF("vptr %p free/total pages %d/%d", ptr, page_context.free_pages, + page_context.total_pages); +} + +/** + * @brief Initialize virtual page allocator + * + * Initializes a virtual page allocator that manages a virtual memory region + * using a page table and block structures. + * + * @retval 0 if successful. + * @retval -ENOMEM on creation failure. + */ +static int init_vpages(void) +{ + const struct sys_mm_drv_region *virtual_memory_regions; + const struct sys_mm_drv_region *region; + uint32_t *bundles = NULL; + size_t block_count, bitmap_num_bundles; + int ret; + + /* Check if already initialized */ + if (vpage_init_done) + return 0; + + /* create the virtual memory region and add it to the system */ + ret = adsp_add_virtual_memory_region(adsp_mm_get_unused_l2_start_aligned(), + CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE, + VIRTUAL_REGION_SHARED_HEAP_ATTR); + if (ret) + return ret; + + memset(&page_context, 0, sizeof(page_context)); + k_mutex_init(&page_context.lock); + + /* now find the virtual region in all memory regions */ + virtual_memory_regions = sys_mm_drv_query_memory_regions(); + SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, region) { + if (region->attr == VIRTUAL_REGION_SHARED_HEAP_ATTR) { + page_context.virtual_region = region; + break; + } + } + sys_mm_drv_query_memory_regions_free(virtual_memory_regions); + + /* check for a valid region */ + if (!page_context.virtual_region) { + LOG_ERR("error: no valid virtual region found"); + return -EINVAL; + } + + block_count = region->size / CONFIG_MM_DRV_PAGE_SIZE; + if (block_count == 0) { + LOG_ERR("error: virtual region too small %d", region->size); + return -ENOMEM; + } + page_context.total_pages = block_count; + page_context.free_pages = block_count; + page_context.num_elems_in_use = 0; + + /* bundles are uint32_t of bits */ + bitmap_num_bundles = SOF_DIV_ROUND_UP(block_count, 32); + + /* allocate memory for bitmap bundles */ + bundles = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + bitmap_num_bundles * sizeof(uint32_t)); + if (!bundles) { + LOG_ERR("error: virtual region bitmap alloc failed"); + return -ENOMEM; + } + + /* Fill allocators data based on config and virtual region data */ + page_context.vpage_blocks.info.num_blocks = block_count; + page_context.vpage_blocks.info.blk_sz_shift = ilog2(CONFIG_MM_DRV_PAGE_SIZE); + /* buffer is the start of the virtual memory region */ + page_context.vpage_blocks.buffer = (uint8_t *)page_context.virtual_region->addr; + + /* initialize bitmap */ + bitmap.num_bits = block_count; + bitmap.num_bundles = bitmap_num_bundles; + bitmap.bundles = bundles; + page_context.vpage_blocks.bitmap = &bitmap; + + LOG_INF("vpage_init region %p size 0x%x pages %d", + (void *)page_context.virtual_region->addr, + (int)page_context.virtual_region->size, block_count); + + vpage_init_done = 1; + return 0; +} + +SYS_INIT(init_vpages, POST_KERNEL, 1); + diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c new file mode 100644 index 000000000000..3651eda785b3 --- /dev/null +++ b/zephyr/lib/vregion.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vregion, CONFIG_SOF_LOG_LEVEL); + +/* + * Pre Allocated Contiguous Virtual Memory Region Allocator + * + * This allocator manages a pre-allocated virtual memory region that uses the + * virtual page allocator to allocate and free memory pages. + * + * It is designed for use cases where a contiguous virtual memory region + * is required, such as for batched allocation of audio pipelines and modules. + * + * New pipelines will create a new virtual region and will specify the size of the region + * which can be divided into multiple areas for different allocation lifetimes, permissions + * and sharing requirements. + * + * Advantages: + * + * 1) Contiguous virtual memory region for easier management and tracking of + * pipeline & DP module memory. i.e. we just need to track the vregion pointer. + * 2) Easier management of memory protection and sharing between different cores + * and domains by partitioning the virtual region into different areas with + * specific permissions and sharing requirements. + * 3) Reduced fragmentation and better cache utilization by using a simple linear + * allocator for lifetime objects. + * + * Note: Software must pass in the size of the region areas at pipeline creation time. + */ + +/** + * @brief virtual region memory region structure. + * + * This structure represents a virtual memory region, which includes + * information about the base address, size, and allocation status + * of the region. + * + * The virtual region memory region can be partitioned into five main areas on + * page-aligned boundaries (some are optional), listed here from base to top: + * + * 1. Text Region (optional): A read-only and executable region that can be used + * to store code or constant data. This region is optional and only present + * if the virtual region is created with a text size. It is page aligned and located + * at the start of the virtual region. Main use case would be DP module text. + * + * 2. Interim Heap: A interim memory area used for multiple temporary allocations + * and frees over the lifetime of the audio processing pipeline. e.g. for + * module kcontrol derived allocations/frees. + * + * 3. Shared Interim Heap (optional): A interim memory area used for multiple temporary + * allocations/frees that can be shared between multiple cores or memory domains. e.g + * shared buffers between different cores or domains. + * + * 4. Lifetime Allocator: A simple incrementing allocator used for long-term static + * allocations that persist for the lifetime of the audio processing pipeline. This + * allocator compresses allocations for better cache utilization. + * + * 5. Shared Lifetime Allocator (optional): A simple incrementing allocator used for long + * term static allocations that can be shared between multiple cores or domains. This + * allocator aligns allocations to cache line boundaries to ensure cache coherency. + * + * * TODO: Pipeline/module reset() could reset the dynamic heap. + */ + + /* linear heap used for lifetime allocations */ +struct vlinear_heap { + uint8_t *base; /* base address of linear allocator */ + uint8_t *ptr; /* current alloc pointer */ + size_t size; /* size of linear allocator in bytes */ + size_t used; /* used bytes in linear allocator */ + int free_count; /* number of frees - tuning only */ +}; + +/* zephyr k_heap for interim allocations. TODO: make lockless for improved performance */ +struct zephyr_heap { + struct k_heap heap; + uint8_t *base; /* base address of zephyr heap allocator */ + size_t size; /* size of heap in bytes */ +}; + +/* Main vregion context, see above intro for more details. + * TODO: Add support to flag which heaps should have their contexts saved and restored. + */ +struct vregion { + /* region context */ + uint8_t *base; /* base address of entire region */ + size_t size; /* size of whole region in bytes */ + size_t pages; /* size of whole region in pages */ + + /* optional text region - RO and Executable */ + struct vlinear_heap text; /* text linear heap */ + + /* interim heap */ + struct zephyr_heap interim; /* interim heap */ + + /* interim shared */ + struct zephyr_heap interim_shared; /* shared interim heap */ + + /* lifetime heap */ + struct vlinear_heap lifetime; /* lifetime linear heap */ + + /* optional shared static buffer heap */ + struct vlinear_heap lifetime_shared; /* shared lifetime linear heap */ +}; + +/** + * @brief Create a new virtual region instance with shared pages. + * + * Create a new VIRTUAL REGION instance with specified static, dynamic, and shared static sizes. + * Total size is the sum of static, dynamic, and shared static sizes. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @param[in] lifetime_shared_size Size of the virtual region shared lifetime partition. + * @param[in] interim_shared_size Size of the virtual region shared interim partition. + * @param[in] text_size Size of the optional read-only text partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size, + size_t lifetime_shared_size, size_t interim_shared_size, + size_t text_size) +{ + struct vregion *vr; + uint32_t pages; + size_t total_size; + uint8_t *vregion_base; + + if (!lifetime_size || !interim_size) { + LOG_ERR("error: invalid vregion lifetime size %d or interim size %d", + lifetime_size, interim_size); + return NULL; + } + + /* + * Align up lifetime sizes and interim sizes to nearest page, the + * vregion structure is stored in lifetime area so account for its size too. + */ + lifetime_size += sizeof(*vr); + lifetime_size = ALIGN_UP(lifetime_size, CONFIG_MM_DRV_PAGE_SIZE); + interim_size = ALIGN_UP(interim_size, CONFIG_MM_DRV_PAGE_SIZE); + lifetime_shared_size = ALIGN_UP(lifetime_shared_size, CONFIG_MM_DRV_PAGE_SIZE); + interim_shared_size = ALIGN_UP(interim_shared_size, CONFIG_MM_DRV_PAGE_SIZE); + text_size = ALIGN_UP(text_size, CONFIG_MM_DRV_PAGE_SIZE); + total_size = lifetime_size + interim_size + + lifetime_shared_size + interim_shared_size + text_size; + + /* allocate pages for vregion */ + pages = total_size / CONFIG_MM_DRV_PAGE_SIZE; + vregion_base = alloc_vpages(pages); + if (!vregion_base) + return NULL; + + /* init vregion - place it at the start of the lifetime region */ + vr = (struct vregion *)(vregion_base + text_size + interim_size); + vr->base = vregion_base; + vr->size = total_size; + vr->pages = pages; + + /* set partition sizes */ + vr->interim.size = interim_size; + vr->interim_shared.size = interim_shared_size; + vr->lifetime.size = lifetime_size; + vr->lifetime_shared.size = lifetime_shared_size; + vr->text.size = text_size; + + /* set base addresses for partitions */ + vr->text.base = vr->base; + vr->interim.base = vr->text.base + text_size; + vr->lifetime.base = vr->interim.base + interim_size; + vr->lifetime_shared.base = vr->lifetime.base + lifetime_size; + vr->interim_shared.base = vr->lifetime_shared.base + lifetime_shared_size; + + /* set alloc ptr addresses for lifetime linear partitions */ + vr->text.ptr = vr->text.base; + vr->lifetime.ptr = vr->lifetime.base + sizeof(*vr); /* skip vregion struct */ + vr->lifetime.used = sizeof(*vr); + vr->lifetime_shared.ptr = vr->lifetime_shared.base; + + /* init interim heaps */ + k_heap_init(&vr->interim.heap, vr->interim.base, interim_size); + if (interim_shared_size) { + k_heap_init(&vr->interim_shared.heap, vr->interim_shared.base, + interim_shared_size); + } + + /* log the new vregion */ + LOG_INF("new at base %p size 0x%x pages %d struct embedded at %p", + (void *)vr->base, total_size, pages, (void *)vr); + LOG_INF(" interim size 0x%x at %p", interim_size, (void *)vr->interim.base); + LOG_INF(" lifetime size 0x%x at %p", lifetime_size, (void *)vr->lifetime.base); + if (interim_shared_size) + LOG_INF(" interim shared size 0x%x at %p", interim_shared_size, + (void *)vr->interim_shared.base); + if (lifetime_shared_size) + LOG_INF(" lifetime shared size 0x%x at %p", lifetime_shared_size, + (void *)vr->lifetime_shared.base); + if (text_size) + LOG_INF(" text size 0x%x at %p", text_size, (void *)vr->text.base); + + return vr; +} + +/** + * @brief Destroy a virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr) +{ + if (!vr) + return; + + /* log the vregion being destroyed */ + LOG_INF("destroy %p size 0x%x pages %d", + (void *)vr->base, vr->size, vr->pages); + LOG_INF(" lifetime used %d free count %d", + vr->lifetime.used, vr->lifetime.free_count); + if (vr->lifetime_shared.size) + LOG_INF(" lifetime shared used %d free count %d", + vr->lifetime_shared.used, vr->lifetime_shared.free_count); + free_vpages(vr->base); +} + +/** + * @brief Allocate memory with alignment from the virtual region dynamic heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *interim_alloc(struct zephyr_heap *heap, + size_t size, size_t align) +{ + void *ptr; + + ptr = k_heap_aligned_alloc(&heap->heap, size, align, K_FOREVER); + if (!ptr) { + LOG_ERR("error: interim alloc failed for %d bytes align %d", + size, align); + return NULL; + } + + return ptr; +} + +/** + * @brief Free memory from the virtual region interim heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void interim_free(struct zephyr_heap *heap, void *ptr) +{ + k_heap_free(&heap->heap, ptr); +} + +/** + * @brief Allocate memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *lifetime_alloc(struct vlinear_heap *heap, + size_t size, size_t align) +{ + void *ptr; + uint8_t *aligned_ptr; + size_t heap_obj_size; + + /* align heap pointer to alignment requested */ + aligned_ptr = UINT_TO_POINTER(ALIGN_UP(POINTER_TO_UINT(heap->ptr), align)); + + /* also align up size to D$ bytes if asked - allocation head and tail aligned */ + if (align == CONFIG_DCACHE_LINE_SIZE) + size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); + + /* calculate new heap object size for object and alignments */ + heap_obj_size = aligned_ptr - heap->ptr + size; + + /* check we have enough lifetime space left */ + if (heap_obj_size + heap->used > heap->size) { + LOG_ERR("error: lifetime alloc failed for object %d heap %d bytes free %d", + size, heap_obj_size, heap->size - heap->used); + return NULL; + } + + /* allocate memory */ + ptr = aligned_ptr; + heap->ptr += heap_obj_size; + heap->used += heap_obj_size; + + return ptr; +} + +/** + * @brief Free memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void lifetime_free(struct vlinear_heap *heap, void *ptr) +{ + /* simple free, just increment free count, this is for tuning only */ + heap->free_count++; + + LOG_DBG("lifetime free %p count %d", ptr, heap->free_count); +} + +/** + * @brief Free memory from the virtual region. + * + * @param vr Pointer to the virtual region instance. + * @param ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr) +{ + if (!vr || !ptr) + return; + + /* check if pointer is in interim heap */ + if (ptr >= (void *)vr->interim.base && + ptr < (void *)(vr->interim.base + vr->interim.size)) { + interim_free(&vr->interim, ptr); + return; + } + + /* check if pointer is in interim shared heap */ + if (vr->interim_shared.size && + ptr >= (void *)vr->interim_shared.base && + ptr < (void *)(vr->interim_shared.base + vr->interim_shared.size)) { + interim_free(&vr->interim_shared, ptr); + return; + } + + /* check if pointer is in lifetime heap */ + if (ptr >= (void *)vr->lifetime.base && + ptr < (void *)(vr->lifetime.base + vr->lifetime.size)) { + lifetime_free(&vr->lifetime, ptr); + return; + } + + /* check if pointer is in lifetime shared heap */ + if (vr->lifetime_shared.size && + ptr >= (void *)vr->lifetime_shared.base && + ptr < (void *)(vr->lifetime_shared.base + vr->lifetime_shared.size)) { + lifetime_free(&vr->lifetime_shared, ptr); + return; + } + + LOG_ERR("error: vregion free invalid pointer %p", ptr); +} + +/** + * @brief Allocate memory type from the virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @param[in] alignment Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment) +{ + if (!vr || !size) + return NULL; + + if (!alignment) + alignment = 4; /* default align 4 bytes */ + + switch (type) { + case VREGION_MEM_TYPE_INTERIM: + return interim_alloc(&vr->interim, size, alignment); + case VREGION_MEM_TYPE_LIFETIME: + return lifetime_alloc(&vr->lifetime, size, alignment); + case VREGION_MEM_TYPE_INTERIM_SHARED: + return interim_alloc(&vr->interim_shared, size, alignment); + case VREGION_MEM_TYPE_LIFETIME_SHARED: + return lifetime_alloc(&vr->lifetime_shared, size, + MAX(alignment, CONFIG_DCACHE_LINE_SIZE)); + default: + LOG_ERR("error: invalid memory type %d", type); + return NULL; + } +} + +/** + * @brief Allocate memory from the virtual region. + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) +{ + return vregion_alloc_align(vr, type, size, 0); +} + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr) +{ + if (!vr) + return; + + LOG_INF("base %p size 0x%x pages %d", + (void *)vr->base, vr->size, vr->pages); + LOG_INF("lifetime used 0x%x free count %d", + vr->lifetime.used, vr->lifetime.free_count); + LOG_INF("lifetime shared used 0x%x free count %d", + vr->lifetime_shared.used, vr->lifetime_shared.free_count); +} +EXPORT_SYMBOL(vregion_info);