From c05526117770a45a1450eef2140d1107b349e499 Mon Sep 17 00:00:00 2001 From: GUI <39894654+GuEe-GUI@users.noreply.github.com> Date: Fri, 22 Nov 2024 14:11:40 +0800 Subject: [PATCH] [DM/FEATURE] Support DMA management (#9682) 1. DMA pool management for platform. 2. DMA engine driver API. Signed-off-by: GuEe-GUI <2991707448@qq.com> --- components/drivers/Kconfig | 1 + components/drivers/dma/Kconfig | 10 + components/drivers/dma/SConscript | 15 + components/drivers/dma/dma.c | 596 ++++++++++++++++++++ components/drivers/dma/dma_pool.c | 685 +++++++++++++++++++++++ components/drivers/include/drivers/dma.h | 235 ++++++++ components/drivers/include/rtdevice.h | 4 + 7 files changed, 1546 insertions(+) create mode 100644 components/drivers/dma/Kconfig create mode 100644 components/drivers/dma/SConscript create mode 100644 components/drivers/dma/dma.c create mode 100644 components/drivers/dma/dma_pool.c create mode 100644 components/drivers/include/drivers/dma.h diff --git a/components/drivers/Kconfig b/components/drivers/Kconfig index 957ed943ad..53158d1353 100755 --- a/components/drivers/Kconfig +++ b/components/drivers/Kconfig @@ -23,6 +23,7 @@ rsource "hwcrypto/Kconfig" rsource "wlan/Kconfig" rsource "block/Kconfig" rsource "virtio/Kconfig" +rsource "dma/Kconfig" rsource "mfd/Kconfig" rsource "ofw/Kconfig" rsource "pci/Kconfig" diff --git a/components/drivers/dma/Kconfig b/components/drivers/dma/Kconfig new file mode 100644 index 0000000000..7bf9d5930f --- /dev/null +++ b/components/drivers/dma/Kconfig @@ -0,0 +1,10 @@ +menuconfig RT_USING_DMA + bool "Using Direct Memory Access (DMA)" + depends on RT_USING_DM + select RT_USING_ADT + select RT_USING_ADT_BITMAP + default n + +if RT_USING_DMA + osource "$(SOC_DM_DMA_DIR)/Kconfig" +endif diff --git a/components/drivers/dma/SConscript b/components/drivers/dma/SConscript new file mode 100644 index 0000000000..ea3500c2c0 --- /dev/null +++ b/components/drivers/dma/SConscript @@ -0,0 +1,15 @@ +from building import * + +group = [] + +if not GetDepend(['RT_USING_DMA']): + Return('group') + +cwd = GetCurrentDir() +CPPPATH = [cwd + '/../include'] + +src = ['dma.c', 'dma_pool.c'] + +group = DefineGroup('DeviceDrivers', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/components/drivers/dma/dma.c b/components/drivers/dma/dma.c new file mode 100644 index 0000000000..7de9d3a096 --- /dev/null +++ b/components/drivers/dma/dma.c @@ -0,0 +1,596 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-02-25 GuEe-GUI the first version + */ + +#include +#include +#include + +#define DBG_TAG "rtdm.dma" +#define DBG_LVL DBG_INFO +#include + +static rt_list_t dmac_nodes = RT_LIST_OBJECT_INIT(dmac_nodes); +static struct rt_spinlock dmac_nodes_lock = {}; + +rt_err_t rt_dma_controller_register(struct rt_dma_controller *ctrl) +{ + const char *dev_name; + char dma_name[RT_NAME_MAX]; + + if (!ctrl || !ctrl->dev || !ctrl->ops) + { + return -RT_EINVAL; + } + + dev_name = rt_dm_dev_get_name(ctrl->dev); + + if (rt_bitmap_next_set_bit(ctrl->dir_cap, 0, RT_DMA_DIR_MAX) == RT_DMA_DIR_MAX) + { + LOG_E("%s: Not direction capability", dev_name); + + return -RT_EINVAL; + } + + rt_snprintf(dma_name, sizeof(dma_name), "%s-dmac", dev_name); + + rt_list_init(&ctrl->list); + + rt_spin_lock(&dmac_nodes_lock); + rt_list_insert_before(&dmac_nodes, &ctrl->list); + rt_spin_unlock(&dmac_nodes_lock); + + rt_list_init(&ctrl->channels_nodes); + rt_mutex_init(&ctrl->mutex, dma_name, RT_IPC_FLAG_PRIO); + + if (ctrl->dev->ofw_node) + { + rt_dm_dev_bind_fwdata(ctrl->dev, RT_NULL, ctrl); + } + + return RT_EOK; +} + +rt_err_t rt_dma_controller_unregister(struct rt_dma_controller *ctrl) +{ + if (!ctrl) + { + return -RT_EINVAL; + } + + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + + if (!rt_list_isempty(&ctrl->channels_nodes)) + { + rt_mutex_release(&ctrl->mutex); + return -RT_EBUSY; + } + + if (ctrl->dev->ofw_node) + { + rt_dm_dev_unbind_fwdata(ctrl->dev, RT_NULL); + } + + rt_mutex_release(&ctrl->mutex); + rt_mutex_detach(&ctrl->mutex); + + rt_spin_lock(&dmac_nodes_lock); + rt_list_remove(&ctrl->list); + rt_spin_unlock(&dmac_nodes_lock); + + return RT_EOK; +} + +rt_err_t rt_dma_chan_start(struct rt_dma_chan *chan) +{ + rt_err_t err; + struct rt_dma_controller *ctrl; + + if (!chan) + { + return -RT_EINVAL; + } + + if (chan->prep_err) + { + LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave)); + + return chan->prep_err; + } + + ctrl = chan->ctrl; + + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + + err = ctrl->ops->start(chan); + + rt_mutex_release(&ctrl->mutex); + + return err; +} + +rt_err_t rt_dma_chan_stop(struct rt_dma_chan *chan) +{ + rt_err_t err; + struct rt_dma_controller *ctrl; + + if (!chan) + { + return -RT_EINVAL; + } + + if (chan->prep_err) + { + LOG_D("%s: Not prepare done", rt_dm_dev_get_name(chan->slave)); + + return chan->prep_err; + } + + ctrl = chan->ctrl; + + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + + err = ctrl->ops->stop(chan); + + rt_mutex_release(&ctrl->mutex); + + return err; +} + +rt_err_t rt_dma_chan_config(struct rt_dma_chan *chan, + struct rt_dma_slave_config *conf) +{ + rt_err_t err; + struct rt_dma_controller *ctrl; + enum rt_dma_transfer_direction dir; + + if (!chan || !conf) + { + err = -RT_EINVAL; + goto _end; + } + + dir = conf->direction; + + if (dir >= RT_DMA_DIR_MAX) + { + err = -RT_EINVAL; + goto _end; + } + + if (conf->src_addr_width >= RT_DMA_SLAVE_BUSWIDTH_BYTES_MAX || + conf->dst_addr_width >= RT_DMA_SLAVE_BUSWIDTH_BYTES_MAX) + { + err = -RT_EINVAL; + goto _end; + } + + ctrl = chan->ctrl; + + if (!rt_bitmap_test_bit(ctrl->dir_cap, dir)) + { + err = -RT_ENOSYS; + goto _end; + } + + if (!chan->name && dir != RT_DMA_MEM_TO_MEM) + { + LOG_E("%s: illegal config for uname channels", + rt_dm_dev_get_name(ctrl->dev)); + + err = -RT_EINVAL; + goto _end; + } + + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + + err = ctrl->ops->config(chan, conf); + + rt_mutex_release(&ctrl->mutex); + + if (!err) + { + rt_memcpy(&chan->conf, conf, sizeof(*conf)); + } + +_end: + chan->conf_err = err; + + return err; +} + +rt_err_t rt_dma_chan_done(struct rt_dma_chan *chan, rt_size_t size) +{ + if (!chan) + { + return -RT_EINVAL; + } + + if (chan->callback) + { + chan->callback(chan, size); + } + + return RT_EOK; +} + +static rt_bool_t range_is_illegal(const char *name, const char *desc, + rt_ubase_t addr0, rt_ubase_t addr1) +{ + rt_bool_t illegal = addr0 < addr1; + + if (illegal) + { + LOG_E("%s: %s %p is out of config %p", name, desc, addr0, addr1); + } + + return illegal; +} + +rt_err_t rt_dma_prep_memcpy(struct rt_dma_chan *chan, + struct rt_dma_slave_transfer *transfer) +{ + rt_err_t err; + rt_size_t len; + rt_ubase_t dma_addr_src, dma_addr_dst; + struct rt_dma_controller *ctrl; + struct rt_dma_slave_config *conf; + + if (!chan || !transfer) + { + return -RT_EINVAL; + } + + ctrl = chan->ctrl; + conf = &chan->conf; + + if (chan->conf_err) + { + LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave)); + + return chan->conf_err; + } + + RT_ASSERT(chan->conf.direction == RT_DMA_MEM_TO_MEM); + dma_addr_src = transfer->src_addr; + dma_addr_dst = transfer->dst_addr; + len = transfer->buffer_len; + + if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "source", + dma_addr_src, conf->src_addr)) + { + return -RT_EINVAL; + } + + if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "dest", + dma_addr_dst, conf->dst_addr)) + { + return -RT_EINVAL; + } + + if (ctrl->ops->prep_memcpy) + { + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + + err = ctrl->ops->prep_memcpy(chan, dma_addr_dst, dma_addr_src, len); + + rt_mutex_release(&ctrl->mutex); + } + else + { + err = -RT_ENOSYS; + } + + if (!err) + { + rt_memcpy(&chan->transfer, transfer, sizeof(*transfer)); + } + + chan->prep_err = err; + + return err; +} + +rt_err_t rt_dma_prep_cyclic(struct rt_dma_chan *chan, + struct rt_dma_slave_transfer *transfer) +{ + rt_err_t err; + rt_ubase_t dma_buf_addr; + struct rt_dma_controller *ctrl; + struct rt_dma_slave_config *conf; + enum rt_dma_transfer_direction dir; + + if (!chan || !transfer) + { + return -RT_EINVAL; + } + + ctrl = chan->ctrl; + conf = &chan->conf; + + if (chan->conf_err) + { + LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave)); + + return chan->conf_err; + } + + dir = chan->conf.direction; + + if (dir == RT_DMA_MEM_TO_DEV || dir == RT_DMA_MEM_TO_MEM) + { + dma_buf_addr = transfer->src_addr; + + if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "source", + dma_buf_addr, conf->src_addr)) + { + return -RT_EINVAL; + } + } + else if (dir == RT_DMA_DEV_TO_MEM) + { + dma_buf_addr = transfer->dst_addr; + + if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "dest", + dma_buf_addr, conf->dst_addr)) + { + return -RT_EINVAL; + } + } + else + { + dma_buf_addr = ~0UL; + } + + if (ctrl->ops->prep_cyclic) + { + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + + err = ctrl->ops->prep_cyclic(chan, dma_buf_addr, + transfer->buffer_len, transfer->period_len, dir); + + rt_mutex_release(&ctrl->mutex); + } + else + { + err = -RT_ENOSYS; + } + + if (!err) + { + rt_memcpy(&chan->transfer, transfer, sizeof(*transfer)); + } + + chan->prep_err = err; + + return err; +} + +rt_err_t rt_dma_prep_single(struct rt_dma_chan *chan, + struct rt_dma_slave_transfer *transfer) +{ + rt_err_t err; + rt_ubase_t dma_buf_addr; + struct rt_dma_controller *ctrl; + struct rt_dma_slave_config *conf; + enum rt_dma_transfer_direction dir; + + if (!chan || !transfer) + { + return -RT_EINVAL; + } + + ctrl = chan->ctrl; + conf = &chan->conf; + + if (chan->conf_err) + { + LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave)); + + return chan->conf_err; + } + + dir = chan->conf.direction; + + if (dir == RT_DMA_MEM_TO_DEV || dir == RT_DMA_MEM_TO_MEM) + { + dma_buf_addr = transfer->src_addr; + + if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "source", + dma_buf_addr, conf->src_addr)) + { + return -RT_EINVAL; + } + } + else if (dir == RT_DMA_DEV_TO_MEM) + { + dma_buf_addr = transfer->dst_addr; + + if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "dest", + dma_buf_addr, conf->dst_addr)) + { + return -RT_EINVAL; + } + } + else + { + dma_buf_addr = ~0UL; + } + + if (ctrl->ops->prep_single) + { + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + + err = ctrl->ops->prep_single(chan, dma_buf_addr, + transfer->buffer_len, dir); + + rt_mutex_release(&ctrl->mutex); + } + else + { + err = -RT_ENOSYS; + } + + if (!err) + { + rt_memcpy(&chan->transfer, transfer, sizeof(*transfer)); + } + + chan->prep_err = err; + + return err; +} + +static struct rt_dma_controller *ofw_find_dma_controller(struct rt_device *dev, + const char *name) +{ + struct rt_dma_controller *ctrl = RT_NULL; +#ifdef RT_USING_OFW + int index; + rt_err_t err; + struct rt_ofw_cell_args dma_args = {}; + struct rt_ofw_node *np = dev->ofw_node, *ctrl_np; + + if (!np) + { + return RT_NULL; + } + + index = rt_ofw_prop_index_of_string(np, "dma-names", name); + + if (index < 0) + { + return RT_NULL; + } + + if (!rt_ofw_parse_phandle_cells(np, "dmas", "#dma-cells", index, &dma_args)) + { + ctrl_np = dma_args.data; + + if (!rt_ofw_data(ctrl_np)) + { + rt_platform_ofw_request(ctrl_np); + } + + ctrl = rt_ofw_data(ctrl_np); + rt_ofw_node_put(ctrl_np); + + if (ctrl && ctrl->ops->ofw_parse) + { + if ((err = ctrl->ops->ofw_parse(ctrl, &dma_args))) + { + ctrl = rt_err_ptr(err); + } + } + } +#endif /* RT_USING_OFW */ + return ctrl; +} + +struct rt_dma_chan *rt_dma_chan_request(struct rt_device *dev, const char *name) +{ + struct rt_dma_chan *chan; + struct rt_dma_controller *ctrl = RT_NULL; + + if (!dev) + { + return rt_err_ptr(-RT_EINVAL); + } + + if (name) + { + ctrl = ofw_find_dma_controller(dev, name); + } + else + { + struct rt_dma_controller *ctrl_tmp; + + rt_spin_lock(&dmac_nodes_lock); + rt_list_for_each_entry(ctrl_tmp, &dmac_nodes, list) + { + /* Only memory to memory for uname request */ + if (rt_bitmap_test_bit(ctrl_tmp->dir_cap, RT_DMA_MEM_TO_MEM)) + { + ctrl = ctrl_tmp; + break; + } + } + rt_spin_unlock(&dmac_nodes_lock); + } + + if (rt_is_err_or_null(ctrl)) + { + return ctrl ? ctrl : rt_err_ptr(-RT_ENOSYS); + } + + if (ctrl->ops->request_chan) + { + chan = ctrl->ops->request_chan(ctrl, dev); + } + else + { + chan = rt_calloc(1, sizeof(*chan)); + + if (!chan) + { + chan = rt_err_ptr(-RT_ENOMEM); + } + } + + if (rt_is_err(chan)) + { + return chan; + } + + if (!chan) + { + LOG_E("%s: unset request channels error", rt_dm_dev_get_name(ctrl->dev)); + + return rt_err_ptr(-RT_ERROR); + } + + chan->name = name; + chan->ctrl = ctrl; + chan->slave = dev; + + rt_list_init(&chan->list); + chan->conf_err = -RT_ERROR; + chan->prep_err = -RT_ERROR; + + rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER); + rt_list_insert_before(&ctrl->channels_nodes, &chan->list); + rt_mutex_release(&ctrl->mutex); + + return chan; +} + +rt_err_t rt_dma_chan_release(struct rt_dma_chan *chan) +{ + rt_err_t err = RT_EOK; + + if (!chan) + { + return -RT_EINVAL; + } + + rt_mutex_take(&chan->ctrl->mutex, RT_WAITING_FOREVER); + rt_list_remove(&chan->list); + rt_mutex_release(&chan->ctrl->mutex); + + if (chan->ctrl->ops->release_chan) + { + err = chan->ctrl->ops->release_chan(chan); + } + else + { + rt_free(chan); + } + + return err; +} diff --git a/components/drivers/dma/dma_pool.c b/components/drivers/dma/dma_pool.c new file mode 100644 index 0000000000..f851b488c8 --- /dev/null +++ b/components/drivers/dma/dma_pool.c @@ -0,0 +1,685 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-02-25 GuEe-GUI the first version + */ + +#include +#include +#include + +#define DBG_TAG "dma.pool" +#define DBG_LVL DBG_INFO +#include + +#include +#include + +static struct rt_spinlock dma_pools_lock = {}; +static rt_list_t dma_pool_nodes = RT_LIST_OBJECT_INIT(dma_pool_nodes); + +static struct rt_dma_pool *dma_pool_install(rt_region_t *region); + +static void *dma_alloc(struct rt_device *dev, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags); +static void dma_free(struct rt_device *dev, rt_size_t size, + void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags); + +rt_inline void region_pool_lock(void) +{ + rt_hw_spin_lock(&dma_pools_lock.lock); +} + +rt_inline void region_pool_unlock(void) +{ + rt_hw_spin_unlock(&dma_pools_lock.lock); +} + +static rt_err_t dma_map_coherent_sync_out_data(struct rt_device *dev, + void *data, rt_size_t size, rt_ubase_t *dma_handle, rt_ubase_t flags) +{ + if (dma_handle) + { + *dma_handle = (rt_ubase_t)rt_kmem_v2p(data); + } + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, data, size); + + return RT_EOK; +} + +static rt_err_t dma_map_coherent_sync_in_data(struct rt_device *dev, + void *out_data, rt_size_t size, rt_ubase_t dma_handle, rt_ubase_t flags) +{ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_INVALIDATE, out_data, size); + + return RT_EOK; +} + +static const struct rt_dma_map_ops dma_map_coherent_ops = +{ + .sync_out_data = dma_map_coherent_sync_out_data, + .sync_in_data = dma_map_coherent_sync_in_data, +}; + +static rt_err_t dma_map_nocoherent_sync_out_data(struct rt_device *dev, + void *data, rt_size_t size, rt_ubase_t *dma_handle, rt_ubase_t flags) +{ + if (dma_handle) + { + *dma_handle = (rt_ubase_t)rt_kmem_v2p(data); + } + + return RT_EOK; +} + +static rt_err_t dma_map_nocoherent_sync_in_data(struct rt_device *dev, + void *out_data, rt_size_t size, rt_ubase_t dma_handle, rt_ubase_t flags) +{ + return RT_EOK; +} + +static const struct rt_dma_map_ops dma_map_nocoherent_ops = +{ + .sync_out_data = dma_map_nocoherent_sync_out_data, + .sync_in_data = dma_map_nocoherent_sync_in_data, +}; + +#ifdef RT_USING_OFW +rt_inline rt_ubase_t ofw_addr_cpu2dma(struct rt_device *dev, rt_ubase_t addr) +{ + return (rt_ubase_t)rt_ofw_translate_cpu2dma(dev->ofw_node, addr); +} + +rt_inline rt_ubase_t ofw_addr_dma2cpu(struct rt_device *dev, rt_ubase_t addr) +{ + return (rt_ubase_t)rt_ofw_translate_dma2cpu(dev->ofw_node, addr); +} + +static void *ofw_dma_map_alloc(struct rt_device *dev, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags) +{ + void *cpu_addr = dma_alloc(dev, size, dma_handle, flags); + + if (cpu_addr && dma_handle) + { + *dma_handle = ofw_addr_cpu2dma(dev, *dma_handle); + } + + return cpu_addr; +} + +static void ofw_dma_map_free(struct rt_device *dev, rt_size_t size, + void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags) +{ + dma_handle = ofw_addr_dma2cpu(dev, dma_handle); + + dma_free(dev, size, cpu_addr, dma_handle, flags); +} + +static rt_err_t ofw_dma_map_sync_out_data(struct rt_device *dev, + void *data, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags) +{ + rt_err_t err; + + if (flags & RT_DMA_F_NOCACHE) + { + err = dma_map_nocoherent_sync_out_data(dev, data, size, dma_handle, flags); + } + else + { + err = dma_map_coherent_sync_out_data(dev, data, size, dma_handle, flags); + } + + if (!err && dma_handle) + { + *dma_handle = ofw_addr_cpu2dma(dev, *dma_handle); + } + + return err; +} + +static rt_err_t ofw_dma_map_sync_in_data(struct rt_device *dev, + void *out_data, rt_size_t size, + rt_ubase_t dma_handle, rt_ubase_t flags) +{ + dma_handle = ofw_addr_dma2cpu(dev, dma_handle); + + if (flags & RT_DMA_F_NOCACHE) + { + return dma_map_nocoherent_sync_in_data(dev, out_data, size, dma_handle, flags); + } + + return dma_map_coherent_sync_in_data(dev, out_data, size, dma_handle, flags); +} + +static const struct rt_dma_map_ops ofw_dma_map_ops = +{ + .alloc = ofw_dma_map_alloc, + .free = ofw_dma_map_free, + .sync_out_data = ofw_dma_map_sync_out_data, + .sync_in_data = ofw_dma_map_sync_in_data, +}; + +static const struct rt_dma_map_ops *ofw_device_dma_ops(struct rt_device *dev) +{ + rt_err_t err; + int region_nr = 0; + const fdt32_t *cell; + rt_phandle phandle; + rt_region_t region; + struct rt_ofw_prop *prop; + struct rt_dma_pool *dma_pool; + const struct rt_dma_map_ops *ops = RT_NULL; + struct rt_ofw_node *mem_np, *np = dev->ofw_node; + + rt_ofw_foreach_prop_u32(np, "memory-region", prop, cell, phandle) + { + rt_uint64_t addr, size; + + if (!(mem_np = rt_ofw_find_node_by_phandle(phandle))) + { + if (region_nr == 0) + { + return RT_NULL; + } + + break; + } + + if ((err = rt_ofw_get_address(mem_np, 0, &addr, &size))) + { + LOG_E("%s: Read '%s' error = %s", rt_ofw_node_full_name(mem_np), + "memory-region", rt_strerror(err)); + + rt_ofw_node_put(mem_np); + continue; + } + + region.start = addr; + region.end = addr + size; + region.name = rt_dm_dev_get_name(dev); + + rt_ofw_node_put(mem_np); + + if (!(dma_pool = dma_pool_install(®ion))) + { + return RT_NULL; + } + + if (rt_ofw_prop_read_bool(mem_np, "no-map")) + { + dma_pool->flags |= RT_DMA_F_NOMAP; + } + + if (!rt_dma_device_is_coherent(dev)) + { + dma_pool->flags |= RT_DMA_F_NOCACHE; + } + + dma_pool->dev = dev; + ++region_nr; + } + + if (region_nr) + { + ops = &ofw_dma_map_ops; + } + + return ops; +} +#endif /* RT_USING_OFW */ + +static const struct rt_dma_map_ops *device_dma_ops(struct rt_device *dev) +{ + const struct rt_dma_map_ops *ops = dev->dma_ops; + + if (ops) + { + return ops; + } + +#ifdef RT_USING_OFW + if (dev->ofw_node && (ops = ofw_device_dma_ops(dev))) + { + return ops; + } +#endif + + if (rt_dma_device_is_coherent(dev)) + { + ops = &dma_map_coherent_ops; + } + else + { + ops = &dma_map_nocoherent_ops; + } + + dev->dma_ops = ops; + + return ops; +} + +static rt_ubase_t dma_pool_alloc(struct rt_dma_pool *pool, rt_size_t size) +{ + rt_size_t bit, next_bit, end_bit, max_bits; + + size = RT_DIV_ROUND_UP(size, ARCH_PAGE_SIZE); + max_bits = pool->bits - size; + + rt_bitmap_for_each_clear_bit(pool->map, bit, max_bits) + { + end_bit = bit + size; + + for (next_bit = bit + 1; next_bit < end_bit; ++next_bit) + { + if (rt_bitmap_test_bit(pool->map, next_bit)) + { + bit = next_bit; + goto _next; + } + } + + if (next_bit == end_bit) + { + while (next_bit --> bit) + { + rt_bitmap_set_bit(pool->map, next_bit); + } + + return pool->start + bit * ARCH_PAGE_SIZE; + } + _next: + } + + return RT_NULL; +} + +static void dma_pool_free(struct rt_dma_pool *pool, rt_ubase_t offset, rt_size_t size) +{ + rt_size_t bit = (offset - pool->start) / ARCH_PAGE_SIZE, end_bit; + + size = RT_DIV_ROUND_UP(size, ARCH_PAGE_SIZE); + end_bit = bit + size; + + for (; bit < end_bit; ++bit) + { + rt_bitmap_clear_bit(pool->map, bit); + } +} + +static void *dma_alloc(struct rt_device *dev, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags) +{ + void *dma_buffer = RT_NULL; + struct rt_dma_pool *pool; + + region_pool_lock(); + + rt_list_for_each_entry(pool, &dma_pool_nodes, list) + { + if ((flags & RT_DMA_F_DEVICE) && + (!(pool->flags & RT_DMA_F_DEVICE) || pool->dev != dev)) + { + continue; + } + + if ((flags & RT_DMA_F_NOMAP) && !((pool->flags & RT_DMA_F_NOMAP))) + { + continue; + } + + if ((flags & RT_DMA_F_32BITS) && !((pool->flags & RT_DMA_F_32BITS))) + { + continue; + } + + if ((flags & RT_DMA_F_LINEAR) && !((pool->flags & RT_DMA_F_LINEAR))) + { + continue; + } + + *dma_handle = dma_pool_alloc(pool, size); + + if (*dma_handle && !(flags & RT_DMA_F_NOMAP)) + { + if (flags & RT_DMA_F_NOCACHE) + { + dma_buffer = rt_ioremap_nocache((void *)*dma_handle, size); + } + else + { + dma_buffer = rt_ioremap_cached((void *)*dma_handle, size); + } + + if (!dma_buffer) + { + dma_pool_free(pool, *dma_handle, size); + + continue; + } + + break; + } + else if (*dma_handle) + { + dma_buffer = (void *)*dma_handle; + + break; + } + } + + region_pool_unlock(); + + return dma_buffer; +} + +static void dma_free(struct rt_device *dev, rt_size_t size, + void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags) +{ + struct rt_dma_pool *pool; + + region_pool_lock(); + + rt_list_for_each_entry(pool, &dma_pool_nodes, list) + { + if (dma_handle >= pool->region.start && + dma_handle <= pool->region.end) + { + rt_iounmap(cpu_addr); + + dma_pool_free(pool, dma_handle, size); + + break; + } + } + + region_pool_unlock(); +} + +void *rt_dma_alloc(struct rt_device *dev, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags) +{ + void *dma_buffer = RT_NULL; + rt_ubase_t dma_handle_s = 0; + const struct rt_dma_map_ops *ops; + + if (!dev || !size) + { + return RT_NULL; + } + + ops = device_dma_ops(dev); + + if (ops->alloc) + { + dma_buffer = ops->alloc(dev, size, &dma_handle_s, flags); + } + else + { + dma_buffer = dma_alloc(dev, size, &dma_handle_s, flags); + } + + if (!dma_buffer) + { + return dma_buffer; + } + + if (dma_handle) + { + *dma_handle = dma_handle_s; + } + + return dma_buffer; +} + +void rt_dma_free(struct rt_device *dev, rt_size_t size, + void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags) +{ + const struct rt_dma_map_ops *ops; + + if (!dev || !size || !cpu_addr) + { + return; + } + + ops = device_dma_ops(dev); + + if (ops->free) + { + ops->free(dev, size, cpu_addr, dma_handle, flags); + } + else + { + dma_free(dev, size, cpu_addr, dma_handle, flags); + } +} + +rt_err_t rt_dma_sync_out_data(struct rt_device *dev, void *data, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags) +{ + rt_err_t err; + rt_ubase_t dma_handle_s = 0; + const struct rt_dma_map_ops *ops; + + if (!data || !size) + { + return -RT_EINVAL; + } + + ops = device_dma_ops(dev); + err = ops->sync_out_data(dev, data, size, &dma_handle_s, flags); + + if (dma_handle) + { + *dma_handle = dma_handle_s; + } + + return err; +} + +rt_err_t rt_dma_sync_in_data(struct rt_device *dev, void *out_data, rt_size_t size, + rt_ubase_t dma_handle, rt_ubase_t flags) +{ + rt_err_t err; + const struct rt_dma_map_ops *ops; + + if (!out_data || !size) + { + return -RT_EINVAL; + } + + ops = device_dma_ops(dev); + err = ops->sync_in_data(dev, out_data, size, dma_handle, flags); + + return err; +} + +static struct rt_dma_pool *dma_pool_install(rt_region_t *region) +{ + rt_err_t err; + struct rt_dma_pool *pool; + + if (!(pool = rt_calloc(1, sizeof(*pool)))) + { + LOG_E("Install pool[%p, %p] error = %s", + region->start, region->end, rt_strerror(-RT_ENOMEM)); + + return RT_NULL; + } + + rt_memcpy(&pool->region, region, sizeof(*region)); + + pool->flags |= RT_DMA_F_LINEAR; + + if (region->end < 4UL * SIZE_GB) + { + pool->flags |= RT_DMA_F_32BITS; + } + + pool->start = RT_ALIGN(pool->region.start, ARCH_PAGE_SIZE); + pool->bits = (pool->region.end - pool->start) / ARCH_PAGE_SIZE; + + if (!pool->bits) + { + err = -RT_EINVAL; + goto _fail; + } + + pool->map = rt_calloc(RT_BITMAP_LEN(pool->bits), sizeof(*pool->map)); + + if (!pool->map) + { + err = -RT_ENOMEM; + goto _fail; + } + + rt_list_init(&pool->list); + + region_pool_lock(); + rt_list_insert_before(&dma_pool_nodes, &pool->list); + region_pool_unlock(); + + return pool; + +_fail: + rt_free(pool); + + LOG_E("Install pool[%p, %p] error = %s", + region->start, region->end, rt_strerror(err)); + + return RT_NULL; +} + +struct rt_dma_pool *rt_dma_pool_install(rt_region_t *region) +{ + struct rt_dma_pool *pool; + + if (!region) + { + return RT_NULL; + } + + if ((pool = dma_pool_install(region))) + { + region = &pool->region; + + LOG_I("%s: Reserved %u.%u MiB at %p", + region->name, + (region->end - region->start) / SIZE_MB, + (region->end - region->start) / SIZE_KB & (SIZE_KB - 1), + region->start); + } + + return pool; +} + +rt_err_t rt_dma_pool_extract(rt_region_t *region_list, rt_size_t list_len, + rt_size_t cma_size, rt_size_t coherent_pool_size) +{ + struct rt_dma_pool *pool; + rt_region_t *region = region_list, *region_high = RT_NULL, cma, coherent_pool; + + if (!region_list || !list_len || cma_size < coherent_pool_size) + { + return -RT_EINVAL; + } + + for (rt_size_t i = 0; i < list_len; ++i, ++region) + { + if (!region->name) + { + continue; + } + + /* Always use low address in 4G */ + if (region->end - region->start >= cma_size) + { + if ((rt_ssize_t)((4UL * SIZE_GB) - region->start) < cma_size) + { + region_high = region; + continue; + } + + goto _found; + } + } + + if (region_high) + { + region = region_high; + LOG_W("No available DMA zone in 4G"); + + goto _found; + } + + return -RT_EEMPTY; + +_found: + if (region->end - region->start != cma_size) + { + cma.start = region->start; + cma.end = cma.start + cma_size; + + /* Update input region */ + region->start += cma_size; + } + else + { + rt_memcpy(&cma, region, sizeof(cma)); + } + + coherent_pool.name = "coherent-pool"; + coherent_pool.start = cma.start; + coherent_pool.end = coherent_pool.start + coherent_pool_size; + + cma.name = "cma"; + cma.start += coherent_pool_size; + + if (!(pool = rt_dma_pool_install(&coherent_pool))) + { + return -RT_ENOMEM; + } + + /* Use: CMA > coherent-pool */ + if (!(pool = rt_dma_pool_install(&cma))) + { + return -RT_ENOMEM; + } + + return RT_EOK; +} + +#if defined(RT_USING_CONSOLE) && defined(RT_USING_MSH) +static int list_dma_pool(int argc, char**argv) +{ + int count = 0; + rt_region_t *region; + struct rt_dma_pool *pool; + + rt_kprintf("%-*.s Region\n", RT_NAME_MAX, "Name"); + + region_pool_lock(); + + rt_list_for_each_entry(pool, &dma_pool_nodes, list) + { + region = &pool->region; + + rt_kprintf("%-*.s [%p, %p]\n", RT_NAME_MAX, region->name, + region->start, region->end); + + ++count; + } + + rt_kprintf("%d DMA memory found\n", count); + + region_pool_unlock(); + + return 0; +} +MSH_CMD_EXPORT(list_dma_pool, dump all dma memory pool); +#endif /* RT_USING_CONSOLE && RT_USING_MSH */ diff --git a/components/drivers/include/drivers/dma.h b/components/drivers/include/drivers/dma.h new file mode 100644 index 0000000000..16f02c3df7 --- /dev/null +++ b/components/drivers/include/drivers/dma.h @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-02-25 GuEe-GUI the first version + */ + +#ifndef __DMA_H__ +#define __DMA_H__ + +#include +#include +#include +#include + +#include +#include +#include + +struct rt_dma_chan; +struct rt_dma_controller_ops; + +enum rt_dma_transfer_direction +{ + RT_DMA_MEM_TO_MEM, + RT_DMA_MEM_TO_DEV, + RT_DMA_DEV_TO_MEM, + RT_DMA_DEV_TO_DEV, + + RT_DMA_DIR_MAX, +}; + +enum rt_dma_slave_buswidth +{ + RT_DMA_SLAVE_BUSWIDTH_UNDEFINED = 0, + RT_DMA_SLAVE_BUSWIDTH_1_BYTE = 1, + RT_DMA_SLAVE_BUSWIDTH_2_BYTES = 2, + RT_DMA_SLAVE_BUSWIDTH_3_BYTES = 3, + RT_DMA_SLAVE_BUSWIDTH_4_BYTES = 4, + RT_DMA_SLAVE_BUSWIDTH_8_BYTES = 8, + RT_DMA_SLAVE_BUSWIDTH_16_BYTES = 16, + RT_DMA_SLAVE_BUSWIDTH_32_BYTES = 32, + RT_DMA_SLAVE_BUSWIDTH_64_BYTES = 64, + RT_DMA_SLAVE_BUSWIDTH_128_BYTES = 128, + + RT_DMA_SLAVE_BUSWIDTH_BYTES_MAX, +}; + +struct rt_dma_slave_config +{ + enum rt_dma_transfer_direction direction; + enum rt_dma_slave_buswidth src_addr_width; + enum rt_dma_slave_buswidth dst_addr_width; + + rt_ubase_t src_addr; + rt_ubase_t dst_addr; + + rt_uint32_t src_maxburst; + rt_uint32_t dst_maxburst; + rt_uint32_t src_port_window_size; + rt_uint32_t dst_port_window_size; +}; + +struct rt_dma_slave_transfer +{ + rt_ubase_t src_addr; + rt_ubase_t dst_addr; + + void *buffer; + rt_ubase_t dma_handle; + rt_size_t buffer_len; + rt_size_t period_len; +}; + +struct rt_dma_controller +{ + rt_list_t list; + + struct rt_device *dev; + + RT_BITMAP_DECLARE(dir_cap, RT_DMA_DIR_MAX); + const struct rt_dma_controller_ops *ops; + + rt_list_t channels_nodes; + struct rt_mutex mutex; +}; + +struct rt_dma_controller_ops +{ + struct rt_dma_chan *(*request_chan)(struct rt_dma_controller *ctrl, struct rt_device *slave); + rt_err_t (*release_chan)(struct rt_dma_chan *chan); + + rt_err_t (*start)(struct rt_dma_chan *chan); + rt_err_t (*stop)(struct rt_dma_chan *chan); + rt_err_t (*config)(struct rt_dma_chan *chan, struct rt_dma_slave_config *conf); + + rt_err_t (*prep_memcpy)(struct rt_dma_chan *chan, + rt_ubase_t dma_addr_src, rt_ubase_t dma_addr_dst, rt_size_t len); + + rt_err_t (*prep_cyclic)(struct rt_dma_chan *chan, + rt_ubase_t dma_buf_addr, rt_size_t buf_len, rt_size_t period_len, + enum rt_dma_transfer_direction dir); + + rt_err_t (*prep_single)(struct rt_dma_chan *chan, + rt_ubase_t dma_buf_addr, rt_size_t buf_len, + enum rt_dma_transfer_direction dir); + + rt_err_t (*ofw_parse)(struct rt_dma_controller *ctrl, struct rt_ofw_cell_args *dma_args); +}; + +struct rt_dma_chan +{ + const char *name; + + struct rt_dma_controller *ctrl; + struct rt_device *slave; + + rt_list_t list; + rt_err_t conf_err; + rt_err_t prep_err; + struct rt_dma_slave_config conf; + struct rt_dma_slave_transfer transfer; + + void (*callback)(struct rt_dma_chan *chan, rt_size_t size); + + void *priv; +}; + +struct rt_dma_pool +{ + rt_region_t region; + + rt_list_t list; + + rt_ubase_t flags; + + rt_bitmap_t *map; + rt_size_t bits; + rt_ubase_t start; + + struct rt_device *dev; +}; + +struct rt_dma_map_ops +{ + void *(*alloc)(struct rt_device *dev, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags); + void (*free)(struct rt_device *dev, rt_size_t size, + void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags); + rt_err_t (*sync_out_data)(struct rt_device *dev, void *data, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags); + rt_err_t (*sync_in_data)(struct rt_device *dev, void *out_data, rt_size_t size, + rt_ubase_t dma_handle, rt_ubase_t flags); +}; + +rt_inline void rt_dma_controller_add_direction(struct rt_dma_controller *ctrl, + enum rt_dma_transfer_direction dir) +{ + RT_ASSERT(ctrl != RT_NULL); + RT_ASSERT(dir < RT_DMA_DIR_MAX); + + rt_bitmap_set_bit(ctrl->dir_cap, dir); +} + +rt_err_t rt_dma_controller_register(struct rt_dma_controller *ctrl); +rt_err_t rt_dma_controller_unregister(struct rt_dma_controller *ctrl); + +rt_err_t rt_dma_chan_start(struct rt_dma_chan *chan); +rt_err_t rt_dma_chan_stop(struct rt_dma_chan *chan); +rt_err_t rt_dma_chan_config(struct rt_dma_chan *chan, + struct rt_dma_slave_config *conf); +rt_err_t rt_dma_chan_done(struct rt_dma_chan *chan, rt_size_t size); + +rt_err_t rt_dma_prep_memcpy(struct rt_dma_chan *chan, + struct rt_dma_slave_transfer *transfer); +rt_err_t rt_dma_prep_cyclic(struct rt_dma_chan *chan, + struct rt_dma_slave_transfer *transfer); +rt_err_t rt_dma_prep_single(struct rt_dma_chan *chan, + struct rt_dma_slave_transfer *transfer); + +struct rt_dma_chan *rt_dma_chan_request(struct rt_device *dev, const char *name); +rt_err_t rt_dma_chan_release(struct rt_dma_chan *chan); + +#define RT_DMA_F_LINEAR RT_BIT(0) +#define RT_DMA_F_32BITS RT_BIT(1) +#define RT_DMA_F_NOCACHE RT_BIT(2) +#define RT_DMA_F_DEVICE RT_BIT(3) +#define RT_DMA_F_NOMAP RT_BIT(4) + +#define RT_DMA_PAGE_SIZE ARCH_PAGE_SIZE + +void *rt_dma_alloc(struct rt_device *dev, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags); + +void rt_dma_free(struct rt_device *dev, rt_size_t size, + void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags); + +rt_inline void *rt_dma_alloc_coherent(struct rt_device *dev, rt_size_t size, + rt_ubase_t *dma_handle) +{ + return rt_dma_alloc(dev, size, dma_handle, + RT_DMA_F_NOCACHE | RT_DMA_F_LINEAR); +} + +rt_inline void rt_dma_free_coherent(struct rt_device *dev, rt_size_t size, + void *cpu_addr, rt_ubase_t dma_handle) +{ + rt_dma_free(dev, size, cpu_addr, dma_handle, + RT_DMA_F_NOCACHE | RT_DMA_F_LINEAR); +} + +rt_err_t rt_dma_sync_out_data(struct rt_device *dev, void *data, rt_size_t size, + rt_ubase_t *dma_handle, rt_ubase_t flags); +rt_err_t rt_dma_sync_in_data(struct rt_device *dev, void *out_data, rt_size_t size, + rt_ubase_t dma_handle, rt_ubase_t flags); + +rt_inline rt_bool_t rt_dma_device_is_coherent(struct rt_device *dev) +{ + return rt_dm_dev_prop_read_bool(dev, "dma-coherent"); +} + +rt_inline void rt_dma_device_set_ops(struct rt_device *dev, + const struct rt_dma_map_ops *ops) +{ + dev->dma_ops = ops; +} + +struct rt_dma_pool *rt_dma_pool_install(rt_region_t *region); +rt_err_t rt_dma_pool_extract(rt_region_t *region_list, rt_size_t list_len, + rt_size_t cma_size, rt_size_t coherent_pool_size); + +#endif /* __DMA_H__ */ diff --git a/components/drivers/include/rtdevice.h b/components/drivers/include/rtdevice.h index ff7a17734f..f6a3cfb9cf 100644 --- a/components/drivers/include/rtdevice.h +++ b/components/drivers/include/rtdevice.h @@ -49,6 +49,10 @@ extern "C" { #include "drivers/blk.h" #endif +#ifdef RT_USING_DMA +#include "drivers/dma.h" +#endif + #include "drivers/iio.h" #ifdef RT_USING_OFW