ipc-zephyr.c

// SPDX-License-Identifier: BSD-3-Clause
//
// Copyright(c) 2022 Intel Corporation. All rights reserved.
//
// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
//         Keyon Jie <yang.jie@linux.intel.com>
//         Rander Wang <rander.wang@intel.com>
//         Serhiy Katsyuba <serhiy.katsyuba@intel.com>
//         Andrey Borisovich <andrey.borisovich@intel.com>
//         Adrian Warecki <adrian.warecki@intel.com>

#include <autoconf.h>

#include <zephyr/kernel.h>

#include <sof/ipc/common.h>

#include <sof/ipc/schedule.h>
#include <sof/lib/mailbox.h> /* for sw_regs with CONFIG_DEBUG_IPC_COUNTERS */
#include <sof/lib/memory.h>  /* for sw_regs with CONFIG_DEBUG_IPC_COUNTERS */
#if defined(CONFIG_PM)
#include <sof/lib/cpu.h>
#include <zephyr/pm/device.h>
#include <zephyr/pm/state.h>
#include <zephyr/irq.h>
#include <zephyr/pm/policy.h>
#else
#include <sof/lib/pm_runtime.h>
#endif /* CONFIG_PM */
#include <sof/lib/uuid.h>
#include <rtos/wait.h>
#include <sof/list.h>
#include <sof/platform.h>
#include <sof/schedule/edf_schedule.h>
#include <sof/schedule/twb_schedule.h>
#include <sof/schedule/schedule.h>
#include <rtos/task.h>
#include <rtos/spinlock.h>
#include <ipc/header.h>
#include <sof/ipc/msg.h>
#include <ipc4/notification.h>

#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

#ifdef CONFIG_INTEL_ADSP_IPC_OLD_INTERFACE
#include <intel_adsp_ipc.h>
#endif
#include <zephyr/ipc/backends/intel_adsp_host_ipc.h>

SOF_DEFINE_REG_UUID(zipc_task);

LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL);

/**
 * @brief Private data for IPC.
 *
 * Written in interrupt context to service incoming IPC message using registered
 * cAVS IPC Message Handler Callback (message_handler function).
 * Filled with content of TDR and TDD registers.
 * When IPC message is read fills ipc_cmd_hdr.
 */
static uint32_t g_last_data, g_last_ext_data, g_last_payload_length;

static uint32_t *g_last_payload;

struct k_sem *wait_ack_sem;

/**
 * @brief cAVS IPC Message Received Callback function.
 *
 * @return -1 so BUSY on the other side will not be cleared immediately but
 * will remain set until message would have been processed by scheduled task, i.e.
 * until ipc_platform_complete_cmd() call.
 */
static void ipc_receive_cb(const void *data, size_t cb_type, void *priv)
{
	struct intel_adsp_ipc_ept_priv_data *priv_data = priv;

	if (cb_type == INTEL_ADSP_IPC_CB_MSG) {
		const struct intel_adsp_ipc_msg *msg = data;

		struct ipc *ipc = priv_data->priv;

		k_spinlock_key_t key;

		key = k_spin_lock(&ipc->lock);

		g_last_data = msg->data;
		g_last_ext_data = msg->ext_data;
		g_last_payload = msg->payload;
		g_last_payload_length = msg->payload_length;

#if CONFIG_DEBUG_IPC_COUNTERS
		increment_ipc_received_counter();
#endif
		ipc_schedule_process(ipc);

		k_spin_unlock(&ipc->lock, key);

		priv_data->cb_ret = -1;
	} else if (cb_type == INTEL_ADSP_IPC_CB_DONE) {
		if (wait_ack_sem)
			k_sem_give(wait_ack_sem);

		priv_data->cb_ret = INTEL_ADSP_IPC_CB_RET_OKAY;
	}
}

static struct ipc_ept ipc_ept;
static struct intel_adsp_ipc_ept_priv_data ipc_ept_priv_data;
static struct ipc_ept_cfg ipc_ept_cfg = {
	.name = "sof_ipc",
	.cb = {
		.received = ipc_receive_cb,
	},
	.priv = &ipc_ept_priv_data,
};

static void ipc_ept_init(struct ipc *ipc)
{
	const struct device *ipc_dev = INTEL_ADSP_IPC_HOST_DEV;

	ipc_ept_priv_data.priv = ipc;

	ipc_service_register_endpoint(ipc_dev, &ipc_ept, &ipc_ept_cfg);
}

static void ipc_complete(void)
{
	ipc_service_send(&ipc_ept, NULL, INTEL_ADSP_IPC_SEND_DONE);
}

static bool ipc_is_complete(void)
{
	return ipc_service_send(&ipc_ept, NULL, INTEL_ADSP_IPC_SEND_IS_COMPLETE) == 0;
}

static int ipc_send_message(struct intel_adsp_ipc_msg *msg)
{
	return ipc_service_send(&ipc_ept, msg, INTEL_ADSP_IPC_SEND_MSG);
}

void ipc_send_message_emergency(uint32_t data, uint32_t ext_data)
{
	struct intel_adsp_ipc_msg msg = {.data = data, .ext_data = ext_data};

	ipc_service_send(&ipc_ept, &msg, INTEL_ADSP_IPC_SEND_MSG_EMERGENCY);
}

static void ipc_send_message_emergency_with_payload(struct intel_adsp_ipc_msg *msg)
{
	ipc_service_send(&ipc_ept, msg, INTEL_ADSP_IPC_SEND_MSG_EMERGENCY);
}

#ifdef CONFIG_PM_DEVICE
/**
 * @brief IPC device suspend handler callback function.
 * Checks whether device power state should be actually changed.
 *
 * @param dev IPC device.
 * @param arg IPC struct pointer.
 */
static int ipc_device_suspend_handler(const struct device *dev, void *arg)
{
	struct ipc *ipc = (struct ipc *)arg;

	int ret = 0;

	if (!(ipc->task_mask & IPC_TASK_POWERDOWN)) {
		tr_err(&ipc_tr,
		       "ipc task mask not set to IPC_TASK_POWERDOWN. Current value: %u",
		       ipc->task_mask);
		ret = -ENOMSG;
	}

	if (!ipc->pm_prepare_D3) {
		tr_err(&ipc_tr, "power state D3 not requested");
		ret = -EBADMSG;
	}

	if (!list_is_empty(&ipc->msg_list)) {
		struct list_item *slist;
		struct ipc_msg *msg;
		/* The only possible message that can be added during power transition procedure is
		 * SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS.
		 */
		const uint32_t exp_hdr = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS);
		bool only_buff_status = true;

		list_for_item(slist, &ipc->msg_list) {
			msg = container_of(slist, struct ipc_msg, list);

			if (msg->header != exp_hdr)
				only_buff_status = false;
		}

		if (only_buff_status) {
			tr_warn(&ipc_tr, "continuing D3 procedure with the msg in the queue");
		} else {
			tr_err(&ipc_tr, "there are queued IPC messages to be sent");
			ret = -EINPROGRESS;
		}
	}

	if (ret != 0)
		ipc_send_failed_power_transition_response();

	return ret;
}

/**
 * @brief IPC device resume handler callback function.
 * Resets IPC control after context restore.
 *
 * @param dev IPC device.
 * @param arg IPC struct pointer.
 */
static int ipc_device_resume_handler(const struct device *dev, void *arg)
{
	struct ipc *ipc = (struct ipc *)arg;

	ipc_set_drvdata(ipc, NULL);
	ipc->task_mask = 0;
	ipc->pm_prepare_D3 = false;

	/* initialize IPC endpoint */
	ipc_ept_init(ipc);

	/* schedule task */
#if CONFIG_TWB_IPC_TASK
	scheduler_twb_task_init(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
							&ipc_task_ops, ipc, 0, "IPC", ZEPHYR_TWB_STACK_SIZE,
							CONFIG_TWB_THREAD_MEDIUM_PRIORITY, ZEPHYR_TWB_BUDGET_MAX / 2);
#else
	schedule_task_init_edf(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
			       &ipc_task_ops, ipc, 0, 0);
#endif
	return 0;
}
#endif /* CONFIG_PM_DEVICE */

#if CONFIG_DEBUG_IPC_COUNTERS

static inline void increment_ipc_received_counter(void)
{
	static uint32_t ipc_received_counter;

	mailbox_sw_reg_write(SRAM_REG_FW_IPC_RECEIVED_COUNT,
			     ipc_received_counter++);
}

static inline void increment_ipc_processed_counter(void)
{
	static uint32_t ipc_processed_counter;
	uint32_t *uncache_counter = cache_to_uncache(&ipc_processed_counter);

	mailbox_sw_reg_write(SRAM_REG_FW_IPC_PROCESSED_COUNT,
			     (*uncache_counter)++);
}

#endif /* CONFIG_DEBUG_IPC_COUNTERS */

int ipc_platform_compact_read_msg(struct ipc_cmd_hdr *hdr, int words)
{
	uint32_t *chdr = (uint32_t *)hdr;

	/* compact messages are 2 words on CAVS 1.8 onwards */
	if (words != 2)
		return 0;

	chdr[0] = g_last_data;
	chdr[1] = g_last_ext_data;

	return 2; /* number of words read */
}

int ipc_platform_compact_write_msg(struct ipc_cmd_hdr *hdr, int words)
{
	return 0; /* number of words read - not currently used on this platform */
}

enum task_state ipc_platform_do_cmd(struct ipc *ipc)
{
	struct ipc_cmd_hdr *hdr;

	dbg_path_hot_start_watching();

	hdr = ipc_compact_read_msg();

	/* perform command */
	ipc_cmd(hdr);

	dbg_path_hot_stop_watching();

	if (ipc->task_mask & IPC_TASK_POWERDOWN ||
	    ipc_get()->pm_prepare_D3) {
#if defined(CONFIG_PM)
		/**
		 * @note For primary core this function
		 * will only force set lower power state
		 * in power management settings.
		 * Core will enter D3 state after exit
		 * IPC thread during idle.
		 */
		cpu_disable_core(PLATFORM_PRIMARY_CORE_ID);
#else
		/**
		 * @note no return - memory will be
		 * powered off and IPC sent.
		 */
		platform_pm_runtime_power_off();
#endif /* CONFIG_PM  */
	}

	return SOF_TASK_STATE_COMPLETED;
}

void ipc_platform_complete_cmd(struct ipc *ipc)
{
	ARG_UNUSED(ipc);
	ipc_complete();

#if CONFIG_DEBUG_IPC_COUNTERS
	increment_ipc_processed_counter();
#endif
}

int ipc_platform_send_msg(const struct ipc_msg *msg)
{
	if (!ipc_is_complete())
		return -EBUSY;

	struct ipc_cmd_hdr *hdr = ipc_prepare_to_send(msg);
	struct intel_adsp_ipc_msg ipc_drv_msg = {
		.data = hdr->pri,
		.ext_data = hdr->ext,
		.payload = (uintptr_t)msg->tx_data,
		.payload_length = msg->tx_size
	};

	return ipc_send_message(&ipc_drv_msg);
}

void ipc_platform_send_msg_direct(const struct ipc_msg *msg)
{
	/* prepare the message and copy to mailbox */
	struct ipc_cmd_hdr *hdr = ipc_prepare_to_send(msg);
	struct intel_adsp_ipc_msg ipc_drv_msg = {
		.data = hdr->pri,
		.ext_data = hdr->ext,
		.payload = (uintptr_t)msg->tx_data,
		.payload_length = msg->tx_size
	};

	ipc_send_message_emergency_with_payload(&ipc_drv_msg);
}

uint32_t *ipc_access_msg_payload(size_t __unused bytes)
{
	/*
	 * The IPC driver has flushed the
	 * cache on payload buffer, so no action needed here.
	 */
	return g_last_payload;
}

int ipc_platform_poll_is_host_ready(void)
{
	return ipc_is_complete();
}

int platform_ipc_init(struct ipc *ipc)
{
	ipc_set_drvdata(ipc, NULL);

	/* schedule task */
#if CONFIG_TWB_IPC_TASK
	scheduler_twb_task_init(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
							&ipc_task_ops, ipc, 0, "IPC", ZEPHYR_TWB_STACK_SIZE,
							CONFIG_TWB_THREAD_MEDIUM_PRIORITY, ZEPHYR_TWB_BUDGET_MAX / 2);
#else
	schedule_task_init_edf(&ipc->ipc_task, SOF_UUID(zipc_task_uuid),
			       &ipc_task_ops, ipc, 0, 0);
#endif
	/* configure interrupt - work is done internally by Zephyr API */

	/* initialize IPC endpoint */
	ipc_ept_init(ipc);

#ifdef CONFIG_PM
	intel_adsp_ipc_set_suspend_handler(INTEL_ADSP_IPC_HOST_DEV,
					   ipc_device_suspend_handler, ipc);
	intel_adsp_ipc_set_resume_handler(INTEL_ADSP_IPC_HOST_DEV,
					  ipc_device_resume_handler, ipc);
#endif

	return 0;
}

#ifdef CONFIG_INTEL_ADSP_IPC_OLD_INTERFACE
static bool ipc_wait_complete(const struct device *dev, void *arg)
{
	k_sem_give(arg);
	return false;
}
#endif

void ipc_platform_wait_ack(struct ipc *ipc)
{
	static struct k_sem ipc_wait_sem;

	k_sem_init(&ipc_wait_sem, 0, 1);

#ifdef CONFIG_INTEL_ADSP_IPC_OLD_INTERFACE
	intel_adsp_ipc_set_done_handler(INTEL_ADSP_IPC_HOST_DEV, ipc_wait_complete, &ipc_wait_sem);
#else
	wait_ack_sem = &ipc_wait_sem;
#endif

	if (k_sem_take(&ipc_wait_sem, Z_TIMEOUT_MS(10)) == -EAGAIN)
		tr_err(&ipc_tr, "Timeout waiting for host ack!");

#ifdef CONFIG_INTEL_ADSP_IPC_OLD_INTERFACE
	intel_adsp_ipc_set_done_handler(INTEL_ADSP_IPC_HOST_DEV, NULL, NULL);
#else
	wait_ack_sem = NULL;
#endif
}