// SPDX-License-Identifier: GPL-2.0+/* * Copyright (C) 2025 Advanced Micro Devices, Inc. */ #include <linux/iopoll.h> #include "isp4_debug.h"#include "isp4_fw_cmd_resp.h"#include "isp4_hw_reg.h"#include "isp4_interface.h" #define ISP4IF_FW_RESP_RB_IRQ_EN_MASK \ (ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT9_EN_MASK\ | ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT12_EN_MASK) #define ISP4IF_FW_CMD_TIMEOUT (HZ / 2) struct isp4if_rb_config { const char *name; u32 index; u32 reg_rptr; u32 reg_wptr; u32 reg_base_lo; u32 reg_base_hi; u32 reg_size; u32 val_size; u64 base_mc_addr; void *base_sys_addr;}; /* FW cmd ring buffer configuration */static struct isp4if_rb_config isp4if_cmd_rb_config[ISP4IF_STREAM_ID_MAX] = { { .name = "CMD_RB_GBL0", .index = 3, .reg_rptr = ISP_RB_RPTR4, .reg_wptr = ISP_RB_WPTR4, .reg_base_lo = ISP_RB_BASE_LO4, .reg_base_hi = ISP_RB_BASE_HI4, .reg_size = ISP_RB_SIZE4, }, { .name = "CMD_RB_STR1", .index = 0, .reg_rptr = ISP_RB_RPTR1, .reg_wptr = ISP_RB_WPTR1, .reg_base_lo = ISP_RB_BASE_LO1, .reg_base_hi = ISP_RB_BASE_HI1, .reg_size = ISP_RB_SIZE1, }, { .name = "CMD_RB_STR2", .index = 1, .reg_rptr = ISP_RB_RPTR2, .reg_wptr = ISP_RB_WPTR2, .reg_base_lo = ISP_RB_BASE_LO2, .reg_base_hi = ISP_RB_BASE_HI2, .reg_size = ISP_RB_SIZE2, }, { .name = "CMD_RB_STR3", .index = 2, .reg_rptr = ISP_RB_RPTR3, .reg_wptr = ISP_RB_WPTR3, .reg_base_lo = ISP_RB_BASE_LO3, .reg_base_hi = ISP_RB_BASE_HI3, .reg_size = ISP_RB_SIZE3, },}; /* FW resp ring buffer configuration */static struct isp4if_rb_config isp4if_resp_rb_config[ISP4IF_STREAM_ID_MAX] = { { .name = "RES_RB_GBL0", .index = 3, .reg_rptr = ISP_RB_RPTR12, .reg_wptr = ISP_RB_WPTR12, .reg_base_lo = ISP_RB_BASE_LO12, .reg_base_hi = ISP_RB_BASE_HI12, .reg_size = ISP_RB_SIZE12, }, { .name = "RES_RB_STR1", .index = 0, .reg_rptr = ISP_RB_RPTR9, .reg_wptr = ISP_RB_WPTR9, .reg_base_lo = ISP_RB_BASE_LO9, .reg_base_hi = ISP_RB_BASE_HI9, .reg_size = ISP_RB_SIZE9, }, { .name = "RES_RB_STR2", .index = 1, .reg_rptr = ISP_RB_RPTR10, .reg_wptr = ISP_RB_WPTR10, .reg_base_lo = ISP_RB_BASE_LO10, .reg_base_hi = ISP_RB_BASE_HI10, .reg_size = ISP_RB_SIZE10, }, { .name = "RES_RB_STR3", .index = 2, .reg_rptr = ISP_RB_RPTR11, .reg_wptr = ISP_RB_WPTR11, .reg_base_lo = ISP_RB_BASE_LO11, .reg_base_hi = ISP_RB_BASE_HI11, .reg_size = ISP_RB_SIZE11, },}; /* FW log ring buffer configuration */static struct isp4if_rb_config isp4if_log_rb_config = { .name = "LOG_RB", .index = 0, .reg_rptr = ISP_LOG_RB_RPTR0, .reg_wptr = ISP_LOG_RB_WPTR0, .reg_base_lo = ISP_LOG_RB_BASE_LO0, .reg_base_hi = ISP_LOG_RB_BASE_HI0, .reg_size = ISP_LOG_RB_SIZE0,}; static struct isp4if_gpu_mem_info *isp4if_gpu_mem_alloc(struct isp4_interface *ispif, u32 mem_size){ struct isp4if_gpu_mem_info *mem_info; struct device *dev = ispif->dev; int ret; mem_info = kmalloc_obj(*mem_info); if (!mem_info) return NULL; mem_info->mem_size = mem_size; ret = isp_kernel_buffer_alloc(dev, mem_info->mem_size, &mem_info->mem_handle, &mem_info->gpu_mc_addr, &mem_info->sys_addr); if (ret) { kfree(mem_info); return NULL; } return mem_info;} static void isp4if_gpu_mem_free(struct isp4_interface *ispif, struct isp4if_gpu_mem_info **mem_info_ptr){ struct isp4if_gpu_mem_info *mem_info = *mem_info_ptr; if (!mem_info) return; *mem_info_ptr = NULL; isp_kernel_buffer_free(&mem_info->mem_handle, &mem_info->gpu_mc_addr, &mem_info->sys_addr); kfree(mem_info);} static void isp4if_dealloc_fw_gpumem(struct isp4_interface *ispif){ isp4if_gpu_mem_free(ispif, &ispif->fw_mem_pool); isp4if_gpu_mem_free(ispif, &ispif->fw_cmd_resp_buf); isp4if_gpu_mem_free(ispif, &ispif->fw_log_buf); for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++) isp4if_gpu_mem_free(ispif, &ispif->meta_info_buf[i]);} static int isp4if_alloc_fw_gpumem(struct isp4_interface *ispif){ struct device *dev = ispif->dev; ispif->fw_mem_pool = isp4if_gpu_mem_alloc(ispif, ISP4FW_MEMORY_POOL_SIZE); if (!ispif->fw_mem_pool) goto error_no_memory; ispif->fw_cmd_resp_buf = isp4if_gpu_mem_alloc(ispif, ISP4IF_RB_PMBMAP_MEM_SIZE); if (!ispif->fw_cmd_resp_buf) goto error_no_memory; ispif->fw_log_buf = isp4if_gpu_mem_alloc(ispif, ISP4IF_FW_LOG_RINGBUF_SIZE); if (!ispif->fw_log_buf) goto error_no_memory; for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++) { ispif->meta_info_buf[i] = isp4if_gpu_mem_alloc(ispif, ISP4IF_META_INFO_BUF_SIZE); if (!ispif->meta_info_buf[i]) goto error_no_memory; } return 0; error_no_memory: dev_err(dev, "failed to allocate gpu memory\n"); isp4if_dealloc_fw_gpumem(ispif); return -ENOMEM;} static u32 isp4if_compute_check_sum(const void *buf, size_t buf_size){ const u8 *surplus_ptr; const u32 *buffer; u32 checksum = 0; size_t i; buffer = (const u32 *)buf; for (i = 0; i < buf_size / sizeof(u32); i++) checksum += buffer[i]; surplus_ptr = (const u8 *)&buffer[i]; /* add surplus data crc checksum */ for (i = 0; i < buf_size % sizeof(u32); i++) checksum += surplus_ptr[i]; return checksum;} void isp4if_clear_cmdq(struct isp4_interface *ispif){ struct isp4if_cmd_element *buf_node, *tmp_node; LIST_HEAD(free_list); scoped_guard(spinlock, &ispif->cmdq_lock) list_splice_init(&ispif->cmdq, &free_list); list_for_each_entry_safe(buf_node, tmp_node, &free_list, list) kfree(buf_node);} static bool isp4if_is_cmdq_rb_full(struct isp4_interface *ispif, enum isp4if_stream_id stream){ struct isp4if_rb_config *rb_config = &isp4if_cmd_rb_config[stream]; u32 rreg = rb_config->reg_rptr, wreg = rb_config->reg_wptr; u32 len = rb_config->val_size; u32 rd_ptr, wr_ptr; u32 bytes_free; rd_ptr = isp4hw_rreg(ispif->mmio, rreg); wr_ptr = isp4hw_rreg(ispif->mmio, wreg); /* * Read and write pointers are equal, indicating the ring buffer * is empty */ if (wr_ptr == rd_ptr) return false; if (wr_ptr > rd_ptr) bytes_free = len - (wr_ptr - rd_ptr); else bytes_free = rd_ptr - wr_ptr; /* * Ignore one byte from the bytes free to prevent rd_ptr from equaling * wr_ptr when the ring buffer is full, because rd_ptr == wr_ptr is * supposed to indicate that the ring buffer is empty. */ return bytes_free <= sizeof(struct isp4fw_cmd);} struct isp4if_cmd_element *isp4if_rm_cmd_from_cmdq(struct isp4_interface *ispif, u32 seq_num, u32 cmd_id){ struct isp4if_cmd_element *ele; guard(spinlock)(&ispif->cmdq_lock); list_for_each_entry(ele, &ispif->cmdq, list) { if (ele->seq_num == seq_num && ele->cmd_id == cmd_id) { list_del(&ele->list); return ele; } } return NULL;} /* Must check that isp4if_is_cmdq_rb_full() == false before calling */static int isp4if_insert_isp_fw_cmd(struct isp4_interface *ispif, enum isp4if_stream_id stream, const struct isp4fw_cmd *cmd){ struct isp4if_rb_config *rb_config = &isp4if_cmd_rb_config[stream]; u32 rreg = rb_config->reg_rptr, wreg = rb_config->reg_wptr; void *mem_sys = rb_config->base_sys_addr; const u32 cmd_sz = sizeof(*cmd); struct device *dev = ispif->dev; u32 len = rb_config->val_size; const void *src = cmd; u32 rd_ptr, wr_ptr; u32 bytes_to_end; rd_ptr = isp4hw_rreg(ispif->mmio, rreg); wr_ptr = isp4hw_rreg(ispif->mmio, wreg); if (rd_ptr >= len || wr_ptr >= len) { dev_err(dev, "rb invalid: stream=%u(%s), rd=%u, wr=%u, len=%u, cmd_sz=%u\n", stream, isp4dbg_get_if_stream_str(stream), rd_ptr, wr_ptr, len, cmd_sz); return -EINVAL; } bytes_to_end = len - wr_ptr; if (bytes_to_end >= cmd_sz) { /* FW cmd is just a straight copy to the write pointer */ memcpy(mem_sys + wr_ptr, src, cmd_sz); isp4hw_wreg(ispif->mmio, wreg, (wr_ptr + cmd_sz) % len); } else { /* * FW cmd is split because the ring buffer needs to wrap * around */ memcpy(mem_sys + wr_ptr, src, bytes_to_end); memcpy(mem_sys, src + bytes_to_end, cmd_sz - bytes_to_end); isp4hw_wreg(ispif->mmio, wreg, cmd_sz - bytes_to_end); } return 0;} static inline enum isp4if_stream_id isp4if_get_fw_stream(u32 cmd_id){ return ISP4IF_STREAM_ID_1;} static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id, const void *package, u32 package_size, bool sync){ enum isp4if_stream_id stream = isp4if_get_fw_stream(cmd_id); struct isp4if_cmd_element *ele = NULL; struct device *dev = ispif->dev; struct isp4fw_cmd cmd; u32 seq_num; int ret; if (package_size > sizeof(cmd.cmd_param)) { dev_err(dev, "fail pkgsize(%u) > %zu cmd:0x%x, stream %d\n", package_size, sizeof(cmd.cmd_param), cmd_id, stream); return -EINVAL; } /* * The struct will be shared with ISP FW, use memset() to guarantee * padding bits are zeroed, since this is not guaranteed on all * compilers. */ memset(&cmd, 0, sizeof(cmd)); cmd.cmd_id = cmd_id; switch (stream) { case ISP4IF_STREAM_ID_GLOBAL: cmd.cmd_stream_id = ISP4FW_STREAM_ID_INVALID; break; case ISP4IF_STREAM_ID_1: cmd.cmd_stream_id = ISP4FW_STREAM_ID_1; break; default: dev_err(dev, "fail bad stream id %d\n", stream); return -EINVAL; } /* Allocate the sync command object early and outside of the lock */ if (sync) { ele = kmalloc_obj(*ele); if (!ele) return -ENOMEM; /* Get two references: one for the resp thread, one for us */ refcount_set(&ele->refcnt, 2); init_completion(&ele->cmd_done); } if (package && package_size) memcpy(cmd.cmd_param, package, package_size); scoped_guard(mutex, &ispif->isp4if_mutex) { ret = read_poll_timeout(isp4if_is_cmdq_rb_full, ret, !ret, ISP4IF_RB_FULL_SLEEP_US, ISP4IF_RB_FULL_TIMEOUT_US, false, ispif, stream); if (ret) { struct isp4if_rb_config *rb_config = &isp4if_resp_rb_config[stream]; u32 rd_ptr = isp4hw_rreg(ispif->mmio, rb_config->reg_rptr); u32 wr_ptr = isp4hw_rreg(ispif->mmio, rb_config->reg_wptr); dev_err(dev, "failed to get free cmdq slot, stream %s(%d),rd %u, wr %u\n", isp4dbg_get_if_stream_str(stream), stream, rd_ptr, wr_ptr); ret = -ETIMEDOUT; goto free_ele; } seq_num = ispif->host2fw_seq_num++; cmd.cmd_seq_num = seq_num; cmd.cmd_check_sum = isp4if_compute_check_sum(&cmd, sizeof(cmd) - sizeof(u32)); /* * only append the fw cmd to queue when its response needs to * be waited for, currently there are only two such commands, * disable channel and stop stream which are only sent after * close camera */ if (ele) { ele->seq_num = seq_num; ele->cmd_id = cmd_id; scoped_guard(spinlock, &ispif->cmdq_lock) list_add_tail(&ele->list, &ispif->cmdq); } ret = isp4if_insert_isp_fw_cmd(ispif, stream, &cmd); if (ret) { dev_err(dev, "fail for insert_isp_fw_cmd cmd_id %s(0x%08x)\n", isp4dbg_get_cmd_str(cmd_id), cmd_id); goto err_dequeue_ele; } } if (ele) { ret = wait_for_completion_timeout(&ele->cmd_done, ISP4IF_FW_CMD_TIMEOUT); if (!ret) { ret = -ETIMEDOUT; goto err_dequeue_ele; } ret = 0; goto put_ele_ref; } return 0; err_dequeue_ele: /* * Try to remove the command from the queue. If that fails, then it * means the response thread is currently using the object, and we need * to use the refcount to avoid a use-after-free by either side. */ if (ele && isp4if_rm_cmd_from_cmdq(ispif, seq_num, cmd_id)) goto free_ele; put_ele_ref: /* Don't free the command if we didn't put the last reference */ if (ele && !refcount_dec_and_test(&ele->refcnt)) ele = NULL; free_ele: kfree(ele); return ret;} static int isp4if_send_buffer(struct isp4_interface *ispif, struct isp4if_img_buf_info *buf_info){ struct isp4fw_cmd_send_buffer cmd; /* * The struct will be shared with ISP FW, use memset() to guarantee * padding bits are zeroed, since this is not guaranteed on all * compilers. */ memset(&cmd, 0, sizeof(cmd)); cmd.buffer_type = ISP4FW_BUFFER_TYPE_PREVIEW; cmd.buffer.vmid_space.bit.space = ISP4FW_ADDR_SPACE_TYPE_GPU_VA; isp4if_split_addr64(buf_info->planes[0].mc_addr, &cmd.buffer.buf_base_a_lo, &cmd.buffer.buf_base_a_hi); cmd.buffer.buf_size_a = buf_info->planes[0].len; isp4if_split_addr64(buf_info->planes[1].mc_addr, &cmd.buffer.buf_base_b_lo, &cmd.buffer.buf_base_b_hi); cmd.buffer.buf_size_b = buf_info->planes[1].len; isp4if_split_addr64(buf_info->planes[2].mc_addr, &cmd.buffer.buf_base_c_lo, &cmd.buffer.buf_base_c_hi); cmd.buffer.buf_size_c = buf_info->planes[2].len; return isp4if_send_fw_cmd(ispif, ISP4FW_CMD_ID_SEND_BUFFER, &cmd, sizeof(cmd), false);} static void isp4if_init_rb_config(struct isp4_interface *ispif, struct isp4if_rb_config *rb_config){ isp4hw_wreg(ispif->mmio, rb_config->reg_rptr, 0x0); isp4hw_wreg(ispif->mmio, rb_config->reg_wptr, 0x0);Lines 1–500 of 830. Load more
// SPDX-License-Identifier: GPL-2.0+ /* * Copyright (C) 2025 Advanced Micro Devices, Inc. */ #include <linux/iopoll.h> #include "isp4_debug.h" #include "isp4_fw_cmd_resp.h" #include "isp4_hw_reg.h" #include "isp4_interface.h" #define ISP4IF_FW_RESP_RB_IRQ_EN_MASK \ (ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT9_EN_MASK\ | ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT12_EN_MASK) #define ISP4IF_FW_CMD_TIMEOUT (HZ / 2) struct isp4if_rb_config { const char *name; u32 index; u32 reg_rptr; u32 reg_wptr; u32 reg_base_lo; u32 reg_base_hi; u32 reg_size; u32 val_size; u64 base_mc_addr; void *base_sys_addr; }; /* FW cmd ring buffer configuration */ static struct isp4if_rb_config isp4if_cmd_rb_config[ISP4IF_STREAM_ID_MAX] = { { .name = "CMD_RB_GBL0", .index = 3, .reg_rptr = ISP_RB_RPTR4, .reg_wptr = ISP_RB_WPTR4, .reg_base_lo = ISP_RB_BASE_LO4, .reg_base_hi = ISP_RB_BASE_HI4, .reg_size = ISP_RB_SIZE4, }, { .name = "CMD_RB_STR1", .index = 0, .reg_rptr = ISP_RB_RPTR1, .reg_wptr = ISP_RB_WPTR1, .reg_base_lo = ISP_RB_BASE_LO1, .reg_base_hi = ISP_RB_BASE_HI1, .reg_size = ISP_RB_SIZE1, }, { .name = "CMD_RB_STR2", .index = 1, .reg_rptr = ISP_RB_RPTR2, .reg_wptr = ISP_RB_WPTR2, .reg_base_lo = ISP_RB_BASE_LO2, .reg_base_hi = ISP_RB_BASE_HI2, .reg_size = ISP_RB_SIZE2, }, { .name = "CMD_RB_STR3", .index = 2, .reg_rptr = ISP_RB_RPTR3, .reg_wptr = ISP_RB_WPTR3, .reg_base_lo = ISP_RB_BASE_LO3, .reg_base_hi = ISP_RB_BASE_HI3, .reg_size = ISP_RB_SIZE3, }, }; /* FW resp ring buffer configuration */ static struct isp4if_rb_config isp4if_resp_rb_config[ISP4IF_STREAM_ID_MAX] = { { .name = "RES_RB_GBL0", .index = 3, .reg_rptr = ISP_RB_RPTR12, .reg_wptr = ISP_RB_WPTR12, .reg_base_lo = ISP_RB_BASE_LO12, .reg_base_hi = ISP_RB_BASE_HI12, .reg_size = ISP_RB_SIZE12, }, { .name = "RES_RB_STR1", .index = 0, .reg_rptr = ISP_RB_RPTR9, .reg_wptr = ISP_RB_WPTR9, .reg_base_lo = ISP_RB_BASE_LO9, .reg_base_hi = ISP_RB_BASE_HI9, .reg_size = ISP_RB_SIZE9, }, { .name = "RES_RB_STR2", .index = 1, .reg_rptr = ISP_RB_RPTR10, .reg_wptr = ISP_RB_WPTR10, .reg_base_lo = ISP_RB_BASE_LO10, .reg_base_hi = ISP_RB_BASE_HI10, .reg_size = ISP_RB_SIZE10, }, { .name = "RES_RB_STR3", .index = 2, .reg_rptr = ISP_RB_RPTR11, .reg_wptr = ISP_RB_WPTR11, .reg_base_lo = ISP_RB_BASE_LO11, .reg_base_hi = ISP_RB_BASE_HI11, .reg_size = ISP_RB_SIZE11, }, }; /* FW log ring buffer configuration */ static struct isp4if_rb_config isp4if_log_rb_config = { .name = "LOG_RB", .index = 0, .reg_rptr = ISP_LOG_RB_RPTR0, .reg_wptr = ISP_LOG_RB_WPTR0, .reg_base_lo = ISP_LOG_RB_BASE_LO0, .reg_base_hi = ISP_LOG_RB_BASE_HI0, .reg_size = ISP_LOG_RB_SIZE0, }; static struct isp4if_gpu_mem_info * isp4if_gpu_mem_alloc(struct isp4_interface *ispif, u32 mem_size) { struct isp4if_gpu_mem_info *mem_info; struct device *dev = ispif->dev; int ret; mem_info = kmalloc_obj(*mem_info); if (!mem_info) return NULL; mem_info->mem_size = mem_size; ret = isp_kernel_buffer_alloc(dev, mem_info->mem_size, &mem_info->mem_handle, &mem_info->gpu_mc_addr, &mem_info->sys_addr); if (ret) { kfree(mem_info); return NULL; } return mem_info; } static void isp4if_gpu_mem_free(struct isp4_interface *ispif, struct isp4if_gpu_mem_info **mem_info_ptr) { struct isp4if_gpu_mem_info *mem_info = *mem_info_ptr; if (!mem_info) return; *mem_info_ptr = NULL; isp_kernel_buffer_free(&mem_info->mem_handle, &mem_info->gpu_mc_addr, &mem_info->sys_addr); kfree(mem_info); } static void isp4if_dealloc_fw_gpumem(struct isp4_interface *ispif) { isp4if_gpu_mem_free(ispif, &ispif->fw_mem_pool); isp4if_gpu_mem_free(ispif, &ispif->fw_cmd_resp_buf); isp4if_gpu_mem_free(ispif, &ispif->fw_log_buf); for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++) isp4if_gpu_mem_free(ispif, &ispif->meta_info_buf[i]); } static int isp4if_alloc_fw_gpumem(struct isp4_interface *ispif) { struct device *dev = ispif->dev; ispif->fw_mem_pool = isp4if_gpu_mem_alloc(ispif, ISP4FW_MEMORY_POOL_SIZE); if (!ispif->fw_mem_pool) goto error_no_memory; ispif->fw_cmd_resp_buf = isp4if_gpu_mem_alloc(ispif, ISP4IF_RB_PMBMAP_MEM_SIZE); if (!ispif->fw_cmd_resp_buf) goto error_no_memory; ispif->fw_log_buf = isp4if_gpu_mem_alloc(ispif, ISP4IF_FW_LOG_RINGBUF_SIZE); if (!ispif->fw_log_buf) goto error_no_memory; for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++) { ispif->meta_info_buf[i] = isp4if_gpu_mem_alloc(ispif, ISP4IF_META_INFO_BUF_SIZE); if (!ispif->meta_info_buf[i]) goto error_no_memory; } return 0; error_no_memory: dev_err(dev, "failed to allocate gpu memory\n"); isp4if_dealloc_fw_gpumem(ispif); return -ENOMEM; } static u32 isp4if_compute_check_sum(const void *buf, size_t buf_size) { const u8 *surplus_ptr; const u32 *buffer; u32 checksum = 0; size_t i; buffer = (const u32 *)buf; for (i = 0; i < buf_size / sizeof(u32); i++) checksum += buffer[i]; surplus_ptr = (const u8 *)&buffer[i]; /* add surplus data crc checksum */ for (i = 0; i < buf_size % sizeof(u32); i++) checksum += surplus_ptr[i]; return checksum; } void isp4if_clear_cmdq(struct isp4_interface *ispif) { struct isp4if_cmd_element *buf_node, *tmp_node; LIST_HEAD(free_list); scoped_guard(spinlock, &ispif->cmdq_lock) list_splice_init(&ispif->cmdq, &free_list); list_for_each_entry_safe(buf_node, tmp_node, &free_list, list) kfree(buf_node); } static bool isp4if_is_cmdq_rb_full(struct isp4_interface *ispif, enum isp4if_stream_id stream) { struct isp4if_rb_config *rb_config = &isp4if_cmd_rb_config[stream]; u32 rreg = rb_config->reg_rptr, wreg = rb_config->reg_wptr; u32 len = rb_config->val_size; u32 rd_ptr, wr_ptr; u32 bytes_free; rd_ptr = isp4hw_rreg(ispif->mmio, rreg); wr_ptr = isp4hw_rreg(ispif->mmio, wreg); /* * Read and write pointers are equal, indicating the ring buffer * is empty */ if (wr_ptr == rd_ptr) return false; if (wr_ptr > rd_ptr) bytes_free = len - (wr_ptr - rd_ptr); else bytes_free = rd_ptr - wr_ptr; /* * Ignore one byte from the bytes free to prevent rd_ptr from equaling * wr_ptr when the ring buffer is full, because rd_ptr == wr_ptr is * supposed to indicate that the ring buffer is empty. */ return bytes_free <= sizeof(struct isp4fw_cmd); } struct isp4if_cmd_element *isp4if_rm_cmd_from_cmdq(struct isp4_interface *ispif, u32 seq_num, u32 cmd_id) { struct isp4if_cmd_element *ele; guard(spinlock)(&ispif->cmdq_lock); list_for_each_entry(ele, &ispif->cmdq, list) { if (ele->seq_num == seq_num && ele->cmd_id == cmd_id) { list_del(&ele->list); return ele; } } return NULL; } /* Must check that isp4if_is_cmdq_rb_full() == false before calling */ static int isp4if_insert_isp_fw_cmd(struct isp4_interface *ispif, enum isp4if_stream_id stream, const struct isp4fw_cmd *cmd) { struct isp4if_rb_config *rb_config = &isp4if_cmd_rb_config[stream]; u32 rreg = rb_config->reg_rptr, wreg = rb_config->reg_wptr; void *mem_sys = rb_config->base_sys_addr; const u32 cmd_sz = sizeof(*cmd); struct device *dev = ispif->dev; u32 len = rb_config->val_size; const void *src = cmd; u32 rd_ptr, wr_ptr; u32 bytes_to_end; rd_ptr = isp4hw_rreg(ispif->mmio, rreg); wr_ptr = isp4hw_rreg(ispif->mmio, wreg); if (rd_ptr >= len || wr_ptr >= len) { dev_err(dev, "rb invalid: stream=%u(%s), rd=%u, wr=%u, len=%u, cmd_sz=%u\n", stream, isp4dbg_get_if_stream_str(stream), rd_ptr, wr_ptr, len, cmd_sz); return -EINVAL; } bytes_to_end = len - wr_ptr; if (bytes_to_end >= cmd_sz) { /* FW cmd is just a straight copy to the write pointer */ memcpy(mem_sys + wr_ptr, src, cmd_sz); isp4hw_wreg(ispif->mmio, wreg, (wr_ptr + cmd_sz) % len); } else { /* * FW cmd is split because the ring buffer needs to wrap * around */ memcpy(mem_sys + wr_ptr, src, bytes_to_end); memcpy(mem_sys, src + bytes_to_end, cmd_sz - bytes_to_end); isp4hw_wreg(ispif->mmio, wreg, cmd_sz - bytes_to_end); } return 0; } static inline enum isp4if_stream_id isp4if_get_fw_stream(u32 cmd_id) { return ISP4IF_STREAM_ID_1; } static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id, const void *package, u32 package_size, bool sync) { enum isp4if_stream_id stream = isp4if_get_fw_stream(cmd_id); struct isp4if_cmd_element *ele = NULL; struct device *dev = ispif->dev; struct isp4fw_cmd cmd; u32 seq_num; int ret; if (package_size > sizeof(cmd.cmd_param)) { dev_err(dev, "fail pkgsize(%u) > %zu cmd:0x%x, stream %d\n", package_size, sizeof(cmd.cmd_param), cmd_id, stream); return -EINVAL; } /* * The struct will be shared with ISP FW, use memset() to guarantee * padding bits are zeroed, since this is not guaranteed on all * compilers. */ memset(&cmd, 0, sizeof(cmd)); cmd.cmd_id = cmd_id; switch (stream) { case ISP4IF_STREAM_ID_GLOBAL: cmd.cmd_stream_id = ISP4FW_STREAM_ID_INVALID; break; case ISP4IF_STREAM_ID_1: cmd.cmd_stream_id = ISP4FW_STREAM_ID_1; break; default: dev_err(dev, "fail bad stream id %d\n", stream); return -EINVAL; } /* Allocate the sync command object early and outside of the lock */ if (sync) { ele = kmalloc_obj(*ele); if (!ele) return -ENOMEM; /* Get two references: one for the resp thread, one for us */ refcount_set(&ele->refcnt, 2); init_completion(&ele->cmd_done); } if (package && package_size) memcpy(cmd.cmd_param, package, package_size); scoped_guard(mutex, &ispif->isp4if_mutex) { ret = read_poll_timeout(isp4if_is_cmdq_rb_full, ret, !ret, ISP4IF_RB_FULL_SLEEP_US, ISP4IF_RB_FULL_TIMEOUT_US, false, ispif, stream); if (ret) { struct isp4if_rb_config *rb_config = &isp4if_resp_rb_config[stream]; u32 rd_ptr = isp4hw_rreg(ispif->mmio, rb_config->reg_rptr); u32 wr_ptr = isp4hw_rreg(ispif->mmio, rb_config->reg_wptr); dev_err(dev, "failed to get free cmdq slot, stream %s(%d),rd %u, wr %u\n", isp4dbg_get_if_stream_str(stream), stream, rd_ptr, wr_ptr); ret = -ETIMEDOUT; goto free_ele; } seq_num = ispif->host2fw_seq_num++; cmd.cmd_seq_num = seq_num; cmd.cmd_check_sum = isp4if_compute_check_sum(&cmd, sizeof(cmd) - sizeof(u32)); /* * only append the fw cmd to queue when its response needs to * be waited for, currently there are only two such commands, * disable channel and stop stream which are only sent after * close camera */ if (ele) { ele->seq_num = seq_num; ele->cmd_id = cmd_id; scoped_guard(spinlock, &ispif->cmdq_lock) list_add_tail(&ele->list, &ispif->cmdq); } ret = isp4if_insert_isp_fw_cmd(ispif, stream, &cmd); if (ret) { dev_err(dev, "fail for insert_isp_fw_cmd cmd_id %s(0x%08x)\n", isp4dbg_get_cmd_str(cmd_id), cmd_id); goto err_dequeue_ele; } } if (ele) { ret = wait_for_completion_timeout(&ele->cmd_done, ISP4IF_FW_CMD_TIMEOUT); if (!ret) { ret = -ETIMEDOUT; goto err_dequeue_ele; } ret = 0; goto put_ele_ref; } return 0; err_dequeue_ele: /* * Try to remove the command from the queue. If that fails, then it * means the response thread is currently using the object, and we need * to use the refcount to avoid a use-after-free by either side. */ if (ele && isp4if_rm_cmd_from_cmdq(ispif, seq_num, cmd_id)) goto free_ele; put_ele_ref: /* Don't free the command if we didn't put the last reference */ if (ele && !refcount_dec_and_test(&ele->refcnt)) ele = NULL; free_ele: kfree(ele); return ret; } static int isp4if_send_buffer(struct isp4_interface *ispif, struct isp4if_img_buf_info *buf_info) { struct isp4fw_cmd_send_buffer cmd; /* * The struct will be shared with ISP FW, use memset() to guarantee * padding bits are zeroed, since this is not guaranteed on all * compilers. */ memset(&cmd, 0, sizeof(cmd)); cmd.buffer_type = ISP4FW_BUFFER_TYPE_PREVIEW; cmd.buffer.vmid_space.bit.space = ISP4FW_ADDR_SPACE_TYPE_GPU_VA; isp4if_split_addr64(buf_info->planes[0].mc_addr, &cmd.buffer.buf_base_a_lo, &cmd.buffer.buf_base_a_hi); cmd.buffer.buf_size_a = buf_info->planes[0].len; isp4if_split_addr64(buf_info->planes[1].mc_addr, &cmd.buffer.buf_base_b_lo, &cmd.buffer.buf_base_b_hi); cmd.buffer.buf_size_b = buf_info->planes[1].len; isp4if_split_addr64(buf_info->planes[2].mc_addr, &cmd.buffer.buf_base_c_lo, &cmd.buffer.buf_base_c_hi); cmd.buffer.buf_size_c = buf_info->planes[2].len; return isp4if_send_fw_cmd(ispif, ISP4FW_CMD_ID_SEND_BUFFER, &cmd, sizeof(cmd), false); } static void isp4if_init_rb_config(struct isp4_interface *ispif, struct isp4if_rb_config *rb_config) { isp4hw_wreg(ispif->mmio, rb_config->reg_rptr, 0x0); isp4hw_wreg(ispif->mmio, rb_config->reg_wptr, 0x0);