audio: make shared buffers uncached for user-space threads

To ease moving audio code to user-space, modify the logic for allocating
shared buffers so that if a buffer is shared (accessed by multiple
cores), and code is run in user-space, the buffer is allocated uncached.

Similarly, in invalidate and writeback functions, check for matching
conditions and skip the cache operations accordingly.

This approach allows to reuse most of the audio module code in
user-space and even have shared buffers between different user-space
threads running on different cores.

There is no impact to builds without user-space, or when running
modules in user-space without shared buffers.

Signed-off-by: Kai Vehmanen <kai.vehmanen@linux.intel.com>

Author: kv2019i

src/audio/buffers/comp_buffer.c

@@ -24,6 +24,7 @@
 #include <errno.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <zephyr/syscall.h>
 
 LOG_MODULE_REGISTER(buffer, CONFIG_SOF_LOG_LEVEL);
 
@@ -235,6 +236,15 @@ struct comp_buffer *buffer_alloc(size_t size, uint32_t flags, uint32_t align,
 
 	tr_dbg(&buffer_tr, "buffer_alloc()");
 
+#if CONFIG_USERSPACE
+	/*
+	 * cache control is not possible in user-space, so cross-core
+	 * buffers must be allocated as coherent
+	 */
+	if (is_shared && k_is_user_context())
+		flags |= SOF_MEM_FLAG_COHERENT;
+#endif
+
 	/* validate request */
 	if (size == 0) {
 		tr_err(&buffer_tr, "new size = %zu is invalid", size);
@@ -273,6 +283,15 @@ struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_siz
 		return NULL;
 	}
 
+#if CONFIG_USERSPACE
+	/*
+	 * cache control is not possible in user-space, so cross-core
+	 * buffers must be allocated as coherent
+	 */
+	if (is_shared && k_is_user_context())
+		flags |= SOF_MEM_FLAG_COHERENT;
+#endif
+
 	/* Align preferred size to a multiple of the minimum size */
 	if (preferred_size % minimum_size)
 		preferred_size += minimum_size - preferred_size % minimum_size;

src/audio/buffers/ring_buffer.c

@@ -13,6 +13,8 @@
 #include <rtos/alloc.h>
 #include <ipc/topology.h>
 
+#include <zephyr/syscall.h>
+
 LOG_MODULE_REGISTER(ring_buffer, CONFIG_SOF_LOG_LEVEL);
 
 SOF_DEFINE_REG_UUID(ring_buffer);
@@ -53,6 +55,14 @@ static inline void ring_buffer_invalidate_shared(struct ring_buffer *ring_buffer
 	if (!ring_buffer_is_shared(ring_buffer))
 		return;
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (k_is_user_context()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(ptr) == false);
+		return;
+	}
+#endif
+
 	/* wrap-around? */
 	if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) {
 		/* writeback till the end of circular buffer */
@@ -72,6 +82,14 @@ static inline void ring_buffer_writeback_shared(struct ring_buffer *ring_buffer,
 	if (!ring_buffer_is_shared(ring_buffer))
 		return;
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (k_is_user_context()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(ptr) == false);
+		return;
+	}
+#endif
+
 	/* wrap-around? */
 	if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) {
 		/* writeback till the end of circular buffer */
@@ -355,6 +373,15 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl
 	 */
 	ring_buffer->data_buffer_size = 3 * max_ibs_obs;
 
+#if CONFIG_USERSPACE
+	/*
+	 * cache control is not possible in user-space, so cross-core
+	 * buffers must be allocated as coherent
+	 */
+	if (is_shared && k_is_user_context())
+		memory_flags |= SOF_MEM_FLAG_COHERENT;
+#endif
+
 	/* allocate data buffer - always in cached memory alias */
 	ring_buffer->data_buffer_size =
 			ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN);

src/include/sof/audio/audio_stream.h

@@ -26,6 +26,7 @@
 #include <ipc/stream.h>
 #include <ipc4/base-config.h>
 #include <module/audio/audio_stream.h>
+#include <zephyr/syscall.h>
 
 #include <stdbool.h>
 #include <stdint.h>
@@ -733,6 +734,14 @@ static inline void audio_stream_invalidate(struct audio_stream *buffer, uint32_t
 		tail_size = bytes - head_size;
 	}
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (k_is_user_context()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(buffer->addr) == false);
+		return;
+	}
+#endif
+
 	dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->r_ptr, head_size);
 	if (tail_size)
 		dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->addr,
@@ -756,6 +765,14 @@ static inline void audio_stream_writeback(struct audio_stream *buffer, uint32_t
 		tail_size = bytes - head_size;
 	}
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (k_is_user_context()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(buffer->addr) == false);
+		return;
+	}
+#endif
+
 	dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, head_size);
 	if (tail_size)
 		dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->addr,

src/include/sof/audio/buffer.h

@@ -27,6 +27,7 @@
 #include <ipc/topology.h>
 #include <user/trace.h>
 #include <sof/audio/format.h>
+#include <zephyr/syscall.h>
 
 #include <stdbool.h>
 #include <stddef.h>
@@ -257,20 +258,32 @@ bool buffer_params_match(struct comp_buffer *buffer,
 static inline void buffer_stream_invalidate(struct comp_buffer *buffer, uint32_t bytes)
 {
 #if CONFIG_INCOHERENT
-	if (audio_buffer_is_shared(&buffer->audio_buffer))
+	if (audio_buffer_is_shared(&buffer->audio_buffer)) {
+#if CONFIG_USERSPACE
+		/* user-space shared buffers are allocated as uncached */
+		if (k_is_user_context())
+			return;
+#endif
 		audio_stream_invalidate(&buffer->stream, bytes);
+	}
 #endif
 }
 
 static inline void buffer_stream_writeback(struct comp_buffer *buffer, uint32_t bytes)
 {
 #if CONFIG_INCOHERENT
-	if (audio_buffer_is_shared(&buffer->audio_buffer))
+	if (audio_buffer_is_shared(&buffer->audio_buffer)) {
+#if CONFIG_USERSPACE
+		/* user-space shared buffers are allocated as uncached */
+		if (k_is_user_context())
+			return;
+#endif
+
 		audio_stream_writeback(&buffer->stream, bytes);
+	}
 #endif
 }
 
-
 /*
  * Attach a new buffer at the beginning of the list. Note, that "head" must
  * really be the head of the list, not a list head within another buffer. We