From 76c33d27073e29bd98d1c975265e1cbe0889fc53 Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Wed, 20 Jun 2018 00:56:59 +0800 Subject: [PATCH 01/12] dmaengine: imx-sdma: factor out a struct sdma_desc from struct sdma_channel This is a preparation step to make the adding of virt-dma easier. We create a struct sdma_desc, move some fields from struct sdma_channel there and add a pointer from the former to the latter. For now we allocate the data statically in struct sdma_channel, but with virt-dma support it will be dynamically allocated. Signed-off-by: Sascha Hauer Signed-off-by: Robin Gong Reviewed-by: Sascha Hauer Tested-by: Lucas Stach Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 137 +++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 54 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index f077992635c2..19c351f3b4bc 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -288,6 +288,30 @@ struct sdma_context_data { struct sdma_engine; +/** + * struct sdma_desc - descriptor structor for one transfer + * @vd descriptor for virt dma + * @num_bd max NUM_BD. number of descriptors currently handling + * @buf_tail ID of the buffer that was processed + * @buf_ptail ID of the previous buffer that was processed + * @period_len period length, used in cyclic. + * @chn_real_count the real count updated from bd->mode.count + * @chn_count the transfer count setuped + * @sdmac sdma_channel pointer + * @bd pointer of alloced bd + */ +struct sdma_desc { + unsigned int num_bd; + dma_addr_t bd_phys; + unsigned int buf_tail; + unsigned int buf_ptail; + unsigned int period_len; + unsigned int chn_real_count; + unsigned int chn_count; + struct sdma_channel *sdmac; + struct sdma_buffer_descriptor *bd; +}; + /** * struct sdma_channel - housekeeping for a SDMA channel * @@ -298,11 +322,10 @@ struct sdma_engine; * @event_id0 aka dma request line * @event_id1 for channels that use 2 events * @word_size peripheral access size - * @buf_tail ID of the buffer that was processed - * @buf_ptail ID of the previous buffer that was processed - * @num_bd max NUM_BD. number of descriptors currently handling */ struct sdma_channel { + struct sdma_desc *desc; + struct sdma_desc _desc; struct sdma_engine *sdma; unsigned int channel; enum dma_transfer_direction direction; @@ -310,12 +333,6 @@ struct sdma_channel { unsigned int event_id0; unsigned int event_id1; enum dma_slave_buswidth word_size; - unsigned int buf_tail; - unsigned int buf_ptail; - unsigned int num_bd; - unsigned int period_len; - struct sdma_buffer_descriptor *bd; - dma_addr_t bd_phys; unsigned int pc_from_device, pc_to_device; unsigned int device_to_device; unsigned long flags; @@ -325,10 +342,8 @@ struct sdma_channel { u32 shp_addr, per_addr; struct dma_chan chan; spinlock_t lock; - struct dma_async_tx_descriptor desc; + struct dma_async_tx_descriptor txdesc; enum dma_status status; - unsigned int chn_count; - unsigned int chn_real_count; struct tasklet_struct tasklet; struct imx_dma_data data; bool enabled; @@ -391,6 +406,8 @@ struct sdma_engine { u32 spba_start_addr; u32 spba_end_addr; unsigned int irq; + dma_addr_t bd0_phys; + struct sdma_buffer_descriptor *bd0; }; static struct sdma_driver_data sdma_imx31 = { @@ -625,7 +642,7 @@ static int sdma_run_channel0(struct sdma_engine *sdma) static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, u32 address) { - struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd; + struct sdma_buffer_descriptor *bd0 = sdma->bd0; void *buf_virt; dma_addr_t buf_phys; int ret; @@ -700,7 +717,9 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * call callback function. */ while (1) { - bd = &sdmac->bd[sdmac->buf_tail]; + struct sdma_desc *desc = sdmac->desc; + + bd = &desc->bd[desc->buf_tail]; if (bd->mode.status & BD_DONE) break; @@ -716,11 +735,11 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * the number of bytes present in the current buffer descriptor. */ - sdmac->chn_real_count = bd->mode.count; + desc->chn_real_count = bd->mode.count; bd->mode.status |= BD_DONE; - bd->mode.count = sdmac->period_len; - sdmac->buf_ptail = sdmac->buf_tail; - sdmac->buf_tail = (sdmac->buf_tail + 1) % sdmac->num_bd; + bd->mode.count = desc->period_len; + desc->buf_ptail = desc->buf_tail; + desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd; /* * The callback is called from the interrupt context in order @@ -729,7 +748,7 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * executed. */ - dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL); + dmaengine_desc_get_callback_invoke(&sdmac->txdesc, NULL); if (error) sdmac->status = old_status; @@ -742,17 +761,17 @@ static void mxc_sdma_handle_channel_normal(unsigned long data) struct sdma_buffer_descriptor *bd; int i, error = 0; - sdmac->chn_real_count = 0; + sdmac->desc->chn_real_count = 0; /* * non loop mode. Iterate over all descriptors, collect * errors and call callback function */ - for (i = 0; i < sdmac->num_bd; i++) { - bd = &sdmac->bd[i]; + for (i = 0; i < sdmac->desc->num_bd; i++) { + bd = &sdmac->desc->bd[i]; if (bd->mode.status & (BD_DONE | BD_RROR)) error = -EIO; - sdmac->chn_real_count += bd->mode.count; + sdmac->desc->chn_real_count += bd->mode.count; } if (error) @@ -760,9 +779,9 @@ static void mxc_sdma_handle_channel_normal(unsigned long data) else sdmac->status = DMA_COMPLETE; - dma_cookie_complete(&sdmac->desc); + dma_cookie_complete(&sdmac->txdesc); - dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL); + dmaengine_desc_get_callback_invoke(&sdmac->txdesc, NULL); } static irqreturn_t sdma_int_handler(int irq, void *dev_id) @@ -890,7 +909,7 @@ static int sdma_load_context(struct sdma_channel *sdmac) int channel = sdmac->channel; int load_address; struct sdma_context_data *context = sdma->context; - struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd; + struct sdma_buffer_descriptor *bd0 = sdma->bd0; int ret; unsigned long flags; @@ -1093,18 +1112,22 @@ static int sdma_set_channel_priority(struct sdma_channel *sdmac, static int sdma_request_channel(struct sdma_channel *sdmac) { struct sdma_engine *sdma = sdmac->sdma; + struct sdma_desc *desc; int channel = sdmac->channel; int ret = -EBUSY; - sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys, + sdmac->desc = &sdmac->_desc; + desc = sdmac->desc; + + desc->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &desc->bd_phys, GFP_KERNEL); - if (!sdmac->bd) { + if (!desc->bd) { ret = -ENOMEM; goto out; } - sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys; - sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; + sdma->channel_control[channel].base_bd_ptr = desc->bd_phys; + sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY); return 0; @@ -1169,10 +1192,10 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) if (ret) goto disable_clk_ahb; - dma_async_tx_descriptor_init(&sdmac->desc, chan); - sdmac->desc.tx_submit = sdma_tx_submit; + dma_async_tx_descriptor_init(&sdmac->txdesc, chan); + sdmac->txdesc.tx_submit = sdma_tx_submit; /* txd.flags will be overwritten in prep funcs */ - sdmac->desc.flags = DMA_CTRL_ACK; + sdmac->txdesc.flags = DMA_CTRL_ACK; return 0; @@ -1187,6 +1210,7 @@ static void sdma_free_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; + struct sdma_desc *desc = sdmac->desc; sdma_disable_channel(chan); @@ -1200,7 +1224,7 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdma_set_channel_priority(sdmac, 0); - dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys); + dma_free_coherent(NULL, PAGE_SIZE, desc->bd, desc->bd_phys); clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); @@ -1216,6 +1240,7 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( int ret, i, count; int channel = sdmac->channel; struct scatterlist *sg; + struct sdma_desc *desc = sdmac->desc; if (sdmac->status == DMA_IN_PROGRESS) return NULL; @@ -1223,9 +1248,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( sdmac->flags = 0; - sdmac->buf_tail = 0; - sdmac->buf_ptail = 0; - sdmac->chn_real_count = 0; + desc->buf_tail = 0; + desc->buf_ptail = 0; + desc->chn_real_count = 0; dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n", sg_len, channel); @@ -1242,9 +1267,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( goto err_out; } - sdmac->chn_count = 0; + desc->chn_count = 0; for_each_sg(sgl, sg, sg_len, i) { - struct sdma_buffer_descriptor *bd = &sdmac->bd[i]; + struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; bd->buffer_addr = sg->dma_address; @@ -1259,7 +1284,7 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( } bd->mode.count = count; - sdmac->chn_count += count; + desc->chn_count += count; if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) { ret = -EINVAL; @@ -1300,10 +1325,10 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( bd->mode.status = param; } - sdmac->num_bd = sg_len; - sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; + desc->num_bd = sg_len; + sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; - return &sdmac->desc; + return &sdmac->txdesc; err_out: sdmac->status = DMA_ERROR; return NULL; @@ -1319,6 +1344,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( int num_periods = buf_len / period_len; int channel = sdmac->channel; int ret, i = 0, buf = 0; + struct sdma_desc *desc = sdmac->desc; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); @@ -1327,10 +1353,10 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( sdmac->status = DMA_IN_PROGRESS; - sdmac->buf_tail = 0; - sdmac->buf_ptail = 0; - sdmac->chn_real_count = 0; - sdmac->period_len = period_len; + desc->buf_tail = 0; + desc->buf_ptail = 0; + desc->chn_real_count = 0; + desc->period_len = period_len; sdmac->flags |= IMX_DMA_SG_LOOP; sdmac->direction = direction; @@ -1351,7 +1377,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( } while (buf < buf_len) { - struct sdma_buffer_descriptor *bd = &sdmac->bd[i]; + struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; bd->buffer_addr = dma_addr; @@ -1382,10 +1408,10 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( i++; } - sdmac->num_bd = num_periods; - sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; + desc->num_bd = num_periods; + sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; - return &sdmac->desc; + return &sdmac->txdesc; err_out: sdmac->status = DMA_ERROR; return NULL; @@ -1424,13 +1450,14 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan, struct dma_tx_state *txstate) { struct sdma_channel *sdmac = to_sdma_chan(chan); + struct sdma_desc *desc = sdmac->desc; u32 residue; if (sdmac->flags & IMX_DMA_SG_LOOP) - residue = (sdmac->num_bd - sdmac->buf_ptail) * - sdmac->period_len - sdmac->chn_real_count; + residue = (desc->num_bd - desc->buf_ptail) * + desc->period_len - desc->chn_real_count; else - residue = sdmac->chn_count - sdmac->chn_real_count; + residue = desc->chn_count - desc->chn_real_count; dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, residue); @@ -1654,6 +1681,8 @@ static int sdma_init(struct sdma_engine *sdma) if (ret) goto err_dma_alloc; + sdma->bd0 = sdma->channel[0].desc->bd; + sdma_config_ownership(&sdma->channel[0], false, true, false); /* Set Command Channel (Channel Zero) */ From 57b772b86871e025c1fc149d8c3e48667be0869f Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 20 Jun 2018 00:57:00 +0800 Subject: [PATCH 02/12] dmaengine: imx-sdma: add virt-dma support The legacy sdma driver has below limitations or drawbacks: 1. Hardcode the max BDs number as "PAGE_SIZE / sizeof(*)", and alloc one page size for one channel regardless of only few BDs needed most time. But in few cases, the max PAGE_SIZE maybe not enough. 2. One SDMA channel can't stop immediatley once channel disabled which means SDMA interrupt may come in after this channel terminated.There are some patches for this corner case such as commit "2746e2c389f9", but not cover non-cyclic. The common virt-dma overcomes the above limitations. It can alloc bd dynamically and free bd once this tx transfer done. No memory wasted or maximum limititation here, only depends on how many memory can be requested from kernel. For No.2, such issue can be workaround by checking if there is available descript("sdmac->desc") now once the unwanted interrupt coming. At last the common virt-dma is easier for sdma driver maintain. Signed-off-by: Robin Gong Reviewed-by: Sascha Hauer Tested-by: Lucas Stach Signed-off-by: Vinod Koul --- drivers/dma/Kconfig | 1 + drivers/dma/imx-sdma.c | 261 ++++++++++++++++++++++++++--------------- 2 files changed, 170 insertions(+), 92 deletions(-) diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index ca1680afa20a..d4a4230a7942 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -250,6 +250,7 @@ config IMX_SDMA tristate "i.MX SDMA support" depends on ARCH_MXC select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS help Support the i.MX SDMA engine. This engine is integrated into Freescale i.MX25/31/35/51/53/6 chips. diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 19c351f3b4bc..86fa7994a5fd 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -41,6 +41,7 @@ #include #include "dmaengine.h" +#include "virt-dma.h" /* SDMA registers */ #define SDMA_H_C0PTR 0x000 @@ -301,6 +302,7 @@ struct sdma_engine; * @bd pointer of alloced bd */ struct sdma_desc { + struct virt_dma_desc vd; unsigned int num_bd; dma_addr_t bd_phys; unsigned int buf_tail; @@ -324,8 +326,8 @@ struct sdma_desc { * @word_size peripheral access size */ struct sdma_channel { + struct virt_dma_chan vc; struct sdma_desc *desc; - struct sdma_desc _desc; struct sdma_engine *sdma; unsigned int channel; enum dma_transfer_direction direction; @@ -340,11 +342,8 @@ struct sdma_channel { unsigned long event_mask[2]; unsigned long watermark_level; u32 shp_addr, per_addr; - struct dma_chan chan; spinlock_t lock; - struct dma_async_tx_descriptor txdesc; enum dma_status status; - struct tasklet_struct tasklet; struct imx_dma_data data; bool enabled; }; @@ -698,6 +697,35 @@ static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) writel_relaxed(val, sdma->regs + chnenbl); } +static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct sdma_desc, vd.tx); +} + +static void sdma_start_desc(struct sdma_channel *sdmac) +{ + struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc); + struct sdma_desc *desc; + struct sdma_engine *sdma = sdmac->sdma; + int channel = sdmac->channel; + + if (!vd) { + sdmac->desc = NULL; + return; + } + sdmac->desc = desc = to_sdma_desc(&vd->tx); + /* + * Do not delete the node in desc_issued list in cyclic mode, otherwise + * the desc alloced will never be freed in vchan_dma_desc_free_list + */ + if (!(sdmac->flags & IMX_DMA_SG_LOOP)) + list_del(&vd->node); + + sdma->channel_control[channel].base_bd_ptr = desc->bd_phys; + sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; + sdma_enable_channel(sdma, sdmac->channel); +} + static void sdma_update_channel_loop(struct sdma_channel *sdmac) { struct sdma_buffer_descriptor *bd; @@ -716,7 +744,7 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * loop mode. Iterate over descriptors, re-setup them and * call callback function. */ - while (1) { + while (sdmac->desc) { struct sdma_desc *desc = sdmac->desc; bd = &desc->bd[desc->buf_tail]; @@ -747,15 +775,16 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * SDMA transaction status by the time the client tasklet is * executed. */ - - dmaengine_desc_get_callback_invoke(&sdmac->txdesc, NULL); + spin_unlock(&sdmac->vc.lock); + dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL); + spin_lock(&sdmac->vc.lock); if (error) sdmac->status = old_status; } } -static void mxc_sdma_handle_channel_normal(unsigned long data) +static void mxc_sdma_handle_channel_normal(struct sdma_channel *data) { struct sdma_channel *sdmac = (struct sdma_channel *) data; struct sdma_buffer_descriptor *bd; @@ -778,10 +807,6 @@ static void mxc_sdma_handle_channel_normal(unsigned long data) sdmac->status = DMA_ERROR; else sdmac->status = DMA_COMPLETE; - - dma_cookie_complete(&sdmac->txdesc); - - dmaengine_desc_get_callback_invoke(&sdmac->txdesc, NULL); } static irqreturn_t sdma_int_handler(int irq, void *dev_id) @@ -797,12 +822,21 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) while (stat) { int channel = fls(stat) - 1; struct sdma_channel *sdmac = &sdma->channel[channel]; + struct sdma_desc *desc; - if (sdmac->flags & IMX_DMA_SG_LOOP) - sdma_update_channel_loop(sdmac); - else - tasklet_schedule(&sdmac->tasklet); + spin_lock(&sdmac->vc.lock); + desc = sdmac->desc; + if (desc) { + if (sdmac->flags & IMX_DMA_SG_LOOP) { + sdma_update_channel_loop(sdmac); + } else { + mxc_sdma_handle_channel_normal(sdmac); + vchan_cookie_complete(&desc->vd); + sdma_start_desc(sdmac); + } + } + spin_unlock(&sdmac->vc.lock); __clear_bit(channel, &stat); } @@ -958,7 +992,7 @@ static int sdma_load_context(struct sdma_channel *sdmac) static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) { - return container_of(chan, struct sdma_channel, chan); + return container_of(chan, struct sdma_channel, vc.chan); } static int sdma_disable_channel(struct dma_chan *chan) @@ -980,7 +1014,16 @@ static int sdma_disable_channel(struct dma_chan *chan) static int sdma_disable_channel_with_delay(struct dma_chan *chan) { + struct sdma_channel *sdmac = to_sdma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + sdma_disable_channel(chan); + spin_lock_irqsave(&sdmac->vc.lock, flags); + vchan_get_all_descriptors(&sdmac->vc, &head); + sdmac->desc = NULL; + spin_unlock_irqrestore(&sdmac->vc.lock, flags); + vchan_dma_desc_free_list(&sdmac->vc, &head); /* * According to NXP R&D team a delay of one BD SDMA cost time @@ -1109,46 +1152,56 @@ static int sdma_set_channel_priority(struct sdma_channel *sdmac, return 0; } -static int sdma_request_channel(struct sdma_channel *sdmac) +static int sdma_request_channel0(struct sdma_engine *sdma) { - struct sdma_engine *sdma = sdmac->sdma; - struct sdma_desc *desc; - int channel = sdmac->channel; int ret = -EBUSY; - sdmac->desc = &sdmac->_desc; - desc = sdmac->desc; - - desc->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &desc->bd_phys, - GFP_KERNEL); - if (!desc->bd) { + sdma->bd0 = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdma->bd0_phys, + GFP_NOWAIT); + if (!sdma->bd0) { ret = -ENOMEM; goto out; } - sdma->channel_control[channel].base_bd_ptr = desc->bd_phys; - sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; + sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys; + sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys; - sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY); + sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY); return 0; out: return ret; } -static dma_cookie_t sdma_tx_submit(struct dma_async_tx_descriptor *tx) + +static int sdma_alloc_bd(struct sdma_desc *desc) { - unsigned long flags; - struct sdma_channel *sdmac = to_sdma_chan(tx->chan); - dma_cookie_t cookie; + u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + int ret = 0; - spin_lock_irqsave(&sdmac->lock, flags); + desc->bd = dma_zalloc_coherent(NULL, bd_size, &desc->bd_phys, + GFP_ATOMIC); + if (!desc->bd) { + ret = -ENOMEM; + goto out; + } +out: + return ret; +} - cookie = dma_cookie_assign(tx); +static void sdma_free_bd(struct sdma_desc *desc) +{ + u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); - spin_unlock_irqrestore(&sdmac->lock, flags); + dma_free_coherent(NULL, bd_size, desc->bd, desc->bd_phys); +} - return cookie; +static void sdma_desc_free(struct virt_dma_desc *vd) +{ + struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd); + + sdma_free_bd(desc); + kfree(desc); } static int sdma_alloc_chan_resources(struct dma_chan *chan) @@ -1184,19 +1237,10 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) if (ret) goto disable_clk_ipg; - ret = sdma_request_channel(sdmac); - if (ret) - goto disable_clk_ahb; - ret = sdma_set_channel_priority(sdmac, prio); if (ret) goto disable_clk_ahb; - dma_async_tx_descriptor_init(&sdmac->txdesc, chan); - sdmac->txdesc.tx_submit = sdma_tx_submit; - /* txd.flags will be overwritten in prep funcs */ - sdmac->txdesc.flags = DMA_CTRL_ACK; - return 0; disable_clk_ahb: @@ -1210,9 +1254,8 @@ static void sdma_free_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - struct sdma_desc *desc = sdmac->desc; - sdma_disable_channel(chan); + sdma_disable_channel_with_delay(chan); if (sdmac->event_id0) sdma_event_disable(sdmac, sdmac->event_id0); @@ -1224,8 +1267,6 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdma_set_channel_priority(sdmac, 0); - dma_free_coherent(NULL, PAGE_SIZE, desc->bd, desc->bd_phys); - clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); } @@ -1240,7 +1281,7 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( int ret, i, count; int channel = sdmac->channel; struct scatterlist *sg; - struct sdma_desc *desc = sdmac->desc; + struct sdma_desc *desc; if (sdmac->status == DMA_IN_PROGRESS) return NULL; @@ -1248,23 +1289,34 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( sdmac->flags = 0; + desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); + if (!desc) + goto err_out; + desc->buf_tail = 0; desc->buf_ptail = 0; + desc->sdmac = sdmac; + desc->num_bd = sg_len; desc->chn_real_count = 0; + if (sdma_alloc_bd(desc)) { + kfree(desc); + goto err_out; + } + dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n", sg_len, channel); sdmac->direction = direction; ret = sdma_load_context(sdmac); if (ret) - goto err_out; + goto err_bd_out; if (sg_len > NUM_BD) { dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n", channel, sg_len, NUM_BD); ret = -EINVAL; - goto err_out; + goto err_bd_out; } desc->chn_count = 0; @@ -1280,7 +1332,7 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n", channel, count, 0xffff); ret = -EINVAL; - goto err_out; + goto err_bd_out; } bd->mode.count = count; @@ -1288,25 +1340,25 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) { ret = -EINVAL; - goto err_out; + goto err_bd_out; } switch (sdmac->word_size) { case DMA_SLAVE_BUSWIDTH_4_BYTES: bd->mode.command = 0; if (count & 3 || sg->dma_address & 3) - return NULL; + goto err_bd_out; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) - return NULL; + goto err_bd_out; break; case DMA_SLAVE_BUSWIDTH_1_BYTE: bd->mode.command = 1; break; default: - return NULL; + goto err_bd_out; } param = BD_DONE | BD_EXTD | BD_CONT; @@ -1325,10 +1377,10 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( bd->mode.status = param; } - desc->num_bd = sg_len; - sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; - - return &sdmac->txdesc; + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); +err_bd_out: + sdma_free_bd(desc); + kfree(desc); err_out: sdmac->status = DMA_ERROR; return NULL; @@ -1344,7 +1396,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( int num_periods = buf_len / period_len; int channel = sdmac->channel; int ret, i = 0, buf = 0; - struct sdma_desc *desc = sdmac->desc; + struct sdma_desc *desc; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); @@ -1353,27 +1405,39 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( sdmac->status = DMA_IN_PROGRESS; + desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); + if (!desc) + goto err_out; + desc->buf_tail = 0; desc->buf_ptail = 0; + desc->sdmac = sdmac; + desc->num_bd = num_periods; desc->chn_real_count = 0; desc->period_len = period_len; sdmac->flags |= IMX_DMA_SG_LOOP; sdmac->direction = direction; + + if (sdma_alloc_bd(desc)) { + kfree(desc); + goto err_bd_out; + } + ret = sdma_load_context(sdmac); if (ret) - goto err_out; + goto err_bd_out; if (num_periods > NUM_BD) { dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n", channel, num_periods, NUM_BD); - goto err_out; + goto err_bd_out; } if (period_len > 0xffff) { dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n", channel, period_len, 0xffff); - goto err_out; + goto err_bd_out; } while (buf < buf_len) { @@ -1385,7 +1449,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( bd->mode.count = period_len; if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) - goto err_out; + goto err_bd_out; if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES) bd->mode.command = 0; else @@ -1408,10 +1472,10 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( i++; } - desc->num_bd = num_periods; - sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; - - return &sdmac->txdesc; + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); +err_bd_out: + sdma_free_bd(desc); + kfree(desc); err_out: sdmac->status = DMA_ERROR; return NULL; @@ -1450,14 +1514,31 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan, struct dma_tx_state *txstate) { struct sdma_channel *sdmac = to_sdma_chan(chan); - struct sdma_desc *desc = sdmac->desc; + struct sdma_desc *desc; u32 residue; + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; - if (sdmac->flags & IMX_DMA_SG_LOOP) - residue = (desc->num_bd - desc->buf_ptail) * - desc->period_len - desc->chn_real_count; - else - residue = desc->chn_count - desc->chn_real_count; + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&sdmac->vc.lock, flags); + vd = vchan_find_desc(&sdmac->vc, cookie); + if (vd) { + desc = to_sdma_desc(&vd->tx); + if (sdmac->flags & IMX_DMA_SG_LOOP) + residue = (desc->num_bd - desc->buf_ptail) * + desc->period_len - desc->chn_real_count; + else + residue = desc->chn_count - desc->chn_real_count; + } else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie) { + residue = sdmac->desc->chn_count - sdmac->desc->chn_real_count; + } else { + residue = 0; + } + spin_unlock_irqrestore(&sdmac->vc.lock, flags); dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, residue); @@ -1468,10 +1549,12 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan, static void sdma_issue_pending(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); - struct sdma_engine *sdma = sdmac->sdma; + unsigned long flags; - if (sdmac->status == DMA_IN_PROGRESS) - sdma_enable_channel(sdma, sdmac->channel); + spin_lock_irqsave(&sdmac->vc.lock, flags); + if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc) + sdma_start_desc(sdmac); + spin_unlock_irqrestore(&sdmac->vc.lock, flags); } #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 @@ -1677,12 +1760,10 @@ static int sdma_init(struct sdma_engine *sdma) for (i = 0; i < MAX_DMA_CHANNELS; i++) writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4); - ret = sdma_request_channel(&sdma->channel[0]); + ret = sdma_request_channel0(sdma); if (ret) goto err_dma_alloc; - sdma->bd0 = sdma->channel[0].desc->bd; - sdma_config_ownership(&sdma->channel[0], false, true, false); /* Set Command Channel (Channel Zero) */ @@ -1843,20 +1924,15 @@ static int sdma_probe(struct platform_device *pdev) sdmac->sdma = sdma; spin_lock_init(&sdmac->lock); - sdmac->chan.device = &sdma->dma_device; - dma_cookie_init(&sdmac->chan); sdmac->channel = i; - - tasklet_init(&sdmac->tasklet, mxc_sdma_handle_channel_normal, - (unsigned long) sdmac); + sdmac->vc.desc_free = sdma_desc_free; /* * Add the channel to the DMAC list. Do not add channel 0 though * because we need it internally in the SDMA driver. This also means * that channel 0 in dmaengine counting matches sdma channel 1. */ if (i) - list_add_tail(&sdmac->chan.device_node, - &sdma->dma_device.channels); + vchan_init(&sdmac->vc, &sdma->dma_device); } ret = sdma_init(sdma); @@ -1961,7 +2037,8 @@ static int sdma_remove(struct platform_device *pdev) for (i = 0; i < MAX_DMA_CHANNELS; i++) { struct sdma_channel *sdmac = &sdma->channel[i]; - tasklet_kill(&sdmac->tasklet); + tasklet_kill(&sdmac->vc.task); + sdma_free_chan_resources(&sdmac->vc.chan); } platform_set_drvdata(pdev, NULL); From d69d1e17c39e0a14f743da72a9e35796e1fd0f87 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 20 Jun 2018 00:57:01 +0800 Subject: [PATCH 03/12] dmaengine: imx-sdma: remove useless 'lock' and 'enabled' in 'struct sdma_channel' Since 'sdmac->vc.lock' and 'sdmac->desc' can be used as 'lock' and 'enabled' in 'struct sdma_channel sdmac', remove them. Signed-off-by: Robin Gong Reviewed-by: Sascha Hauer Tested-by: Lucas Stach Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 86fa7994a5fd..d1d3494eba01 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -342,10 +342,8 @@ struct sdma_channel { unsigned long event_mask[2]; unsigned long watermark_level; u32 shp_addr, per_addr; - spinlock_t lock; enum dma_status status; struct imx_dma_data data; - bool enabled; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -606,14 +604,7 @@ static int sdma_config_ownership(struct sdma_channel *sdmac, static void sdma_enable_channel(struct sdma_engine *sdma, int channel) { - unsigned long flags; - struct sdma_channel *sdmac = &sdma->channel[channel]; - writel(BIT(channel), sdma->regs + SDMA_H_START); - - spin_lock_irqsave(&sdmac->lock, flags); - sdmac->enabled = true; - spin_unlock_irqrestore(&sdmac->lock, flags); } /* @@ -731,14 +722,6 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) struct sdma_buffer_descriptor *bd; int error = 0; enum dma_status old_status = sdmac->status; - unsigned long flags; - - spin_lock_irqsave(&sdmac->lock, flags); - if (!sdmac->enabled) { - spin_unlock_irqrestore(&sdmac->lock, flags); - return; - } - spin_unlock_irqrestore(&sdmac->lock, flags); /* * loop mode. Iterate over descriptors, re-setup them and @@ -1000,15 +983,10 @@ static int sdma_disable_channel(struct dma_chan *chan) struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; int channel = sdmac->channel; - unsigned long flags; writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP); sdmac->status = DMA_ERROR; - spin_lock_irqsave(&sdmac->lock, flags); - sdmac->enabled = false; - spin_unlock_irqrestore(&sdmac->lock, flags); - return 0; } @@ -1922,7 +1900,6 @@ static int sdma_probe(struct platform_device *pdev) struct sdma_channel *sdmac = &sdma->channel[i]; sdmac->sdma = sdma; - spin_lock_init(&sdmac->lock); sdmac->channel = i; sdmac->vc.desc_free = sdma_desc_free; From 36e8d3b133e16eb2aee8a53b9ed5cd34821f6123 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 20 Jun 2018 00:57:02 +0800 Subject: [PATCH 04/12] dmaengine: imx-sdma: remove the maximum limitation for bd numbers No this limitation now after virtual dma used since bd is allocated dynamically instead of static. Signed-off-by: Robin Gong Reviewed-by: Sascha Hauer Tested-by: Lucas Stach Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index d1d3494eba01..9f1a4621fd9a 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -285,7 +285,6 @@ struct sdma_context_data { u32 scratch7; } __attribute__ ((packed)); -#define NUM_BD (int)(PAGE_SIZE / sizeof(struct sdma_buffer_descriptor)) struct sdma_engine; @@ -1290,13 +1289,6 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (ret) goto err_bd_out; - if (sg_len > NUM_BD) { - dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n", - channel, sg_len, NUM_BD); - ret = -EINVAL; - goto err_bd_out; - } - desc->chn_count = 0; for_each_sg(sgl, sg, sg_len, i) { struct sdma_buffer_descriptor *bd = &desc->bd[i]; @@ -1406,12 +1398,6 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( if (ret) goto err_bd_out; - if (num_periods > NUM_BD) { - dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n", - channel, num_periods, NUM_BD); - goto err_bd_out; - } - if (period_len > 0xffff) { dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n", channel, period_len, 0xffff); From 21420841a554dae8a193cd3454ba91d913cfc320 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 20 Jun 2018 00:57:03 +0800 Subject: [PATCH 05/12] dmaengine: imx-sdma: add sdma_transfer_init to decrease code overlap There are lot of codes overlap between prep_sg and prep_cyclic function. Add sdma_transfer_init() function to elimated the code overlap. Signed-off-by: Robin Gong Reviewed-by: Sascha Hauer Tested-by: Lucas Stach Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 83 +++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 9f1a4621fd9a..f8becafd7083 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1248,6 +1248,40 @@ static void sdma_free_chan_resources(struct dma_chan *chan) clk_disable(sdma->clk_ahb); } +static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, + enum dma_transfer_direction direction, u32 bds) +{ + struct sdma_desc *desc; + + desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); + if (!desc) + goto err_out; + + sdmac->status = DMA_IN_PROGRESS; + sdmac->direction = direction; + sdmac->flags = 0; + + desc->chn_count = 0; + desc->chn_real_count = 0; + desc->buf_tail = 0; + desc->buf_ptail = 0; + desc->sdmac = sdmac; + desc->num_bd = bds; + + if (sdma_alloc_bd(desc)) + goto err_desc_out; + + if (sdma_load_context(sdmac)) + goto err_desc_out; + + return desc; + +err_desc_out: + kfree(desc); +err_out: + return NULL; +} + static struct dma_async_tx_descriptor *sdma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, @@ -1260,36 +1294,13 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( struct scatterlist *sg; struct sdma_desc *desc; - if (sdmac->status == DMA_IN_PROGRESS) - return NULL; - sdmac->status = DMA_IN_PROGRESS; - - sdmac->flags = 0; - - desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); + desc = sdma_transfer_init(sdmac, direction, sg_len); if (!desc) goto err_out; - desc->buf_tail = 0; - desc->buf_ptail = 0; - desc->sdmac = sdmac; - desc->num_bd = sg_len; - desc->chn_real_count = 0; - - if (sdma_alloc_bd(desc)) { - kfree(desc); - goto err_out; - } - dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n", sg_len, channel); - sdmac->direction = direction; - ret = sdma_load_context(sdmac); - if (ret) - goto err_bd_out; - - desc->chn_count = 0; for_each_sg(sgl, sg, sg_len, i) { struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; @@ -1365,38 +1376,18 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( struct sdma_engine *sdma = sdmac->sdma; int num_periods = buf_len / period_len; int channel = sdmac->channel; - int ret, i = 0, buf = 0; + int i = 0, buf = 0; struct sdma_desc *desc; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); - if (sdmac->status == DMA_IN_PROGRESS) - return NULL; - - sdmac->status = DMA_IN_PROGRESS; - - desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); + desc = sdma_transfer_init(sdmac, direction, num_periods); if (!desc) goto err_out; - desc->buf_tail = 0; - desc->buf_ptail = 0; - desc->sdmac = sdmac; - desc->num_bd = num_periods; - desc->chn_real_count = 0; desc->period_len = period_len; sdmac->flags |= IMX_DMA_SG_LOOP; - sdmac->direction = direction; - - if (sdma_alloc_bd(desc)) { - kfree(desc); - goto err_bd_out; - } - - ret = sdma_load_context(sdmac); - if (ret) - goto err_bd_out; if (period_len > 0xffff) { dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n", From fe5b85c656bcf54468cb1efcd692a491a524ae86 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 20 Jun 2018 00:57:04 +0800 Subject: [PATCH 06/12] dmaengine: imx-sdma: alloclate bd memory from dma pool dma_terminate_all maybe called in interrupt context which means WARN_ON() will be triggered as below when bd memory freed. Allocat bd memory from dma pool instead. [ 29.161079] WARNING: CPU: 1 PID: 533 at ./include/linux/dma-mapping.h:541 sdma_free_bd+0xa4/0xb4 [ 29.169883] Modules linked in: [ 29.172990] CPU: 1 PID: 533 Comm: mpegaudioparse0 Not tainted 4.18.0-rc1-next-20180618-00009-gf79f22c #20 [ 29.182597] Hardware name: Freescale i.MX6 Quad/DualLite (Device Tree) [ 29.189163] Backtrace: [ 29.191685] [] (dump_backtrace) from [] (show_stack+0x18/0x1c) [ 29.199306] r7:00000000 r6:600f0093 r5:00000000 r4:c107db7c [ 29.205029] [] (show_stack) from [] (dump_stack+0xb4/0xe8) [ 29.212312] [] (dump_stack) from [] (__warn+0x104/0x130) [ 29.219411] r9:ec3e817c r8:0000021d r7:00000009 r6:c0d1d440 r5:00000000 r4:00000000 [ 29.227204] [] (__warn) from [] (warn_slowpath_null+0x44/0x50) [ 29.234821] r8:ed129dc4 r7:c0b01978 r6:c04d4e90 r5:0000021d r4:c0d1d440 [ 29.241574] [] (warn_slowpath_null) from [] (sdma_free_bd+0xa4/0xb4) [ 29.249706] r6:4c001000 r5:f082e000 r4:00000024 [ 29.254376] [] (sdma_free_bd) from [] (sdma_desc_free+0x14/0x20) [ 29.262163] r7:ec3e8110 r6:00000100 r5:00000200 r4:ecf89a00 [ 29.267873] [] (sdma_desc_free) from [] (vchan_dma_desc_free_list+0xa4/0xac) [ 29.276697] r5:00000200 r4:ed129d9c [ 29.280326] [] (vchan_dma_desc_free_list) from [] (sdma_disable_channel_with_delay+0x14c/0x188) [ 29.290808] r9:ecae560c r8:ec3e815c r7:00000000 r6:c1008908 r5:ed129dc4 r4:ec3e8110 [ 29.298605] [] (sdma_disable_channel_with_delay) from [] (snd_dmaengine_pcm_trigger+0x90/0x1b0) [ 29.309087] r8:ecae5000 r7:ec940800 r6:ed31bd80 r5:ecadb200 r4:ec26a700 [ 29.315855] [] (snd_dmaengine_pcm_trigger) from [] (soc_pcm_trigger+0xb4/0x130) [ 29.324953] r8:ecae5000 r7:ec940800 r6:00000000 r5:ecadb200 r4:ec26a700 [ 29.331716] [] (soc_pcm_trigger) from [] (snd_pcm_do_stop+0x58/0x5c) [ 29.339859] r9:ecaed5a8 r8:ed31bdc0 r7:00000000 r6:00000001 r5:ecadb200 r4:c0b9c4d0 [ 29.347652] [] (snd_pcm_do_stop) from [] (snd_pcm_action_single+0x40/0x80) [ 29.356315] [] (snd_pcm_action_single) from [] (snd_pcm_action+0xf4/0xfc) [ 29.364883] r7:00000001 r6:c0b9c4d0 r5:ecadb2d4 r4:ecadb200 [ 29.370593] [] (snd_pcm_action) from [] (snd_pcm_drop+0x58/0x9c) Signed-off-by: Robin Gong Reviewed-by: Sascha Hauer Tested-by: Lucas Stach Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index f8becafd7083..7dab7e949efa 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -343,6 +344,7 @@ struct sdma_channel { u32 shp_addr, per_addr; enum dma_status status; struct imx_dma_data data; + struct dma_pool *bd_pool; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -1153,11 +1155,10 @@ static int sdma_request_channel0(struct sdma_engine *sdma) static int sdma_alloc_bd(struct sdma_desc *desc) { - u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); int ret = 0; - desc->bd = dma_zalloc_coherent(NULL, bd_size, &desc->bd_phys, - GFP_ATOMIC); + desc->bd = dma_pool_alloc(desc->sdmac->bd_pool, GFP_ATOMIC, + &desc->bd_phys); if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1168,9 +1169,7 @@ static int sdma_alloc_bd(struct sdma_desc *desc) static void sdma_free_bd(struct sdma_desc *desc) { - u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); - - dma_free_coherent(NULL, bd_size, desc->bd, desc->bd_phys); + dma_pool_free(desc->sdmac->bd_pool, desc->bd, desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1218,6 +1217,10 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) if (ret) goto disable_clk_ahb; + sdmac->bd_pool = dma_pool_create("bd_pool", chan->device->dev, + sizeof(struct sdma_buffer_descriptor), + 32, 0); + return 0; disable_clk_ahb: @@ -1246,6 +1249,9 @@ static void sdma_free_chan_resources(struct dma_chan *chan) clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); + + dma_pool_destroy(sdmac->bd_pool); + sdmac->bd_pool = NULL; } static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, From 680302c47513f727063031f7ec4fcac0316241eb Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Mon, 2 Jul 2018 18:34:02 +0530 Subject: [PATCH 07/12] dmaengine: imx-sdma: Fix some typos Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 7dab7e949efa..d650065f42dd 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -297,9 +297,9 @@ struct sdma_engine; * @buf_ptail ID of the previous buffer that was processed * @period_len period length, used in cyclic. * @chn_real_count the real count updated from bd->mode.count - * @chn_count the transfer count setuped + * @chn_count the transfer count set * @sdmac sdma_channel pointer - * @bd pointer of alloced bd + * @bd pointer of allocate bd */ struct sdma_desc { struct virt_dma_desc vd; @@ -708,7 +708,7 @@ static void sdma_start_desc(struct sdma_channel *sdmac) sdmac->desc = desc = to_sdma_desc(&vd->tx); /* * Do not delete the node in desc_issued list in cyclic mode, otherwise - * the desc alloced will never be freed in vchan_dma_desc_free_list + * the desc allocated will never be freed in vchan_dma_desc_free_list */ if (!(sdmac->flags & IMX_DMA_SG_LOOP)) list_del(&vd->node); From c1199875d327a9b3bbb3fb1cca0d467d3e625fbb Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Mon, 2 Jul 2018 18:37:27 +0530 Subject: [PATCH 08/12] dmaengine: imx-sdma: Use GFP_NOWAIT for dma allocations The memory allocation in DMA callbacks should use GFP_NOWAIT, so update this one and fix code alignment for this call while at it. Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index d650065f42dd..dfd1fbbe7ba8 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1157,8 +1157,8 @@ static int sdma_alloc_bd(struct sdma_desc *desc) { int ret = 0; - desc->bd = dma_pool_alloc(desc->sdmac->bd_pool, GFP_ATOMIC, - &desc->bd_phys); + desc->bd = dma_pool_alloc(desc->sdmac->bd_pool, GFP_NOWAIT, + &desc->bd_phys); if (!desc->bd) { ret = -ENOMEM; goto out; From ad78b000fc100aec89ebe1ced9ed80e696444123 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Mon, 2 Jul 2018 18:42:51 +0530 Subject: [PATCH 09/12] dmaengine: imx-sdma: remove unused variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The variable is no longer used, though it is set, so remove it drivers/dma/imx-sdma.c:1298:6: warning: variable ‘ret’ set but not used [-Wunused-but-set-variable] int ret, i, count; ^~~ Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index dfd1fbbe7ba8..b78146b4ea01 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1295,7 +1295,7 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - int ret, i, count; + int i, count; int channel = sdmac->channel; struct scatterlist *sg; struct sdma_desc *desc; @@ -1318,17 +1318,14 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (count > 0xffff) { dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n", channel, count, 0xffff); - ret = -EINVAL; goto err_bd_out; } bd->mode.count = count; desc->chn_count += count; - if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) { - ret = -EINVAL; + if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) goto err_bd_out; - } switch (sdmac->word_size) { case DMA_SLAVE_BUSWIDTH_4_BYTES: From 24ca312dd66329b126230b6f5826f45a61ebb5a8 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 4 Jul 2018 18:06:42 +0800 Subject: [PATCH 10/12] dmaengine: imx-sdma: add missing structure description Some member description or colons missing cause build warning with 'W=1' as below: drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'vd' not described in 'sdma_desc' drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'num_bd' not described in 'sdma_desc' drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'bd_phys' not described in 'sdma_desc' drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'buf_tail' not described in 'sdma_desc' drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'buf_ptail' not described in 'sdma_desc' drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'period_len' not described in 'sdma_desc' drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'chn_real_count' not described in 'sdma_desc' drivers/dma/imx-sdma.c:326: warning: Function parameter or member 'chn_count' not described in 'sdma_desc Signed-off-by: Robin Gong Reported-by: Vinod Koul Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 85 ++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index b78146b4ea01..3b622d660c8b 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -202,9 +202,9 @@ struct sdma_buffer_descriptor { /** * struct sdma_channel_control - Channel control Block * - * @current_bd_ptr current buffer descriptor processed - * @base_bd_ptr first element of buffer descriptor array - * @unused padding. The SDMA engine expects an array of 128 byte + * @current_bd_ptr: current buffer descriptor processed + * @base_bd_ptr: first element of buffer descriptor array + * @unused: padding. The SDMA engine expects an array of 128 byte * control blocks */ struct sdma_channel_control { @@ -217,10 +217,13 @@ struct sdma_channel_control { * struct sdma_state_registers - SDMA context for a channel * * @pc: program counter + * @unused1: unused * @t: test bit: status of arithmetic & test instruction * @rpc: return program counter + * @unused0: unused * @sf: source fault while loading data * @spc: loop start program counter + * @unused2: unused * @df: destination fault while storing data * @epc: loop end program counter * @lm: loop mode @@ -258,6 +261,14 @@ struct sdma_state_registers { * @dsa: dedicated core source address register * @ds: dedicated core status register * @dd: dedicated core data register + * @scratch0: 1st word of dedicated ram for context switch + * @scratch1: 2nd word of dedicated ram for context switch + * @scratch2: 3rd word of dedicated ram for context switch + * @scratch3: 4th word of dedicated ram for context switch + * @scratch4: 5th word of dedicated ram for context switch + * @scratch5: 6th word of dedicated ram for context switch + * @scratch6: 7th word of dedicated ram for context switch + * @scratch7: 8th word of dedicated ram for context switch */ struct sdma_context_data { struct sdma_state_registers channel_state; @@ -291,15 +302,16 @@ struct sdma_engine; /** * struct sdma_desc - descriptor structor for one transfer - * @vd descriptor for virt dma - * @num_bd max NUM_BD. number of descriptors currently handling - * @buf_tail ID of the buffer that was processed - * @buf_ptail ID of the previous buffer that was processed - * @period_len period length, used in cyclic. - * @chn_real_count the real count updated from bd->mode.count - * @chn_count the transfer count set - * @sdmac sdma_channel pointer - * @bd pointer of allocate bd + * @vd: descriptor for virt dma + * @num_bd: number of descriptors currently handling + * @bd_phys: physical address of bd + * @buf_tail: ID of the buffer that was processed + * @buf_ptail: ID of the previous buffer that was processed + * @period_len: period length, used in cyclic. + * @chn_real_count: the real count updated from bd->mode.count + * @chn_count: the transfer count set + * @sdmac: sdma_channel pointer + * @bd: pointer of allocate bd */ struct sdma_desc { struct virt_dma_desc vd; @@ -317,13 +329,30 @@ struct sdma_desc { /** * struct sdma_channel - housekeeping for a SDMA channel * - * @sdma pointer to the SDMA engine for this channel - * @channel the channel number, matches dmaengine chan_id + 1 - * @direction transfer type. Needed for setting SDMA script - * @peripheral_type Peripheral type. Needed for setting SDMA script - * @event_id0 aka dma request line - * @event_id1 for channels that use 2 events - * @word_size peripheral access size + * @vc: virt_dma base structure + * @desc: sdma description including vd and other special member + * @sdma: pointer to the SDMA engine for this channel + * @channel: the channel number, matches dmaengine chan_id + 1 + * @direction: transfer type. Needed for setting SDMA script + * @peripheral_type: Peripheral type. Needed for setting SDMA script + * @event_id0: aka dma request line + * @event_id1: for channels that use 2 events + * @word_size: peripheral access size + * @pc_from_device: script address for those device_2_memory + * @pc_to_device: script address for those memory_2_device + * @device_to_device: script address for those device_2_device + * @flags: loop mode or not + * @per_address: peripheral source or destination address in common case + * destination address in p_2_p case + * @per_address2: peripheral source address in p_2_p case + * @event_mask: event mask used in p_2_p script + * @watermark_level: value for gReg[7], some script will extend it from + * basic watermark such as p_2_p + * @shp_addr: value for gReg[6] + * @per_addr: value for gReg[2] + * @status: status of dma channel + * @data: specific sdma interface structure + * @bd_pool: dma_pool for bd */ struct sdma_channel { struct virt_dma_chan vc; @@ -359,15 +388,15 @@ struct sdma_channel { /** * struct sdma_firmware_header - Layout of the firmware image * - * @magic "SDMA" - * @version_major increased whenever layout of struct sdma_script_start_addrs - * changes. - * @version_minor firmware minor version (for binary compatible changes) - * @script_addrs_start offset of struct sdma_script_start_addrs in this image - * @num_script_addrs Number of script addresses in this image - * @ram_code_start offset of SDMA ram image in this firmware image - * @ram_code_size size of SDMA ram image - * @script_addrs Stores the start address of the SDMA scripts + * @magic: "SDMA" + * @version_major: increased whenever layout of struct + * sdma_script_start_addrs changes. + * @version_minor: firmware minor version (for binary compatible changes) + * @script_addrs_start: offset of struct sdma_script_start_addrs in this image + * @num_script_addrs: Number of script addresses in this image + * @ram_code_start: offset of SDMA ram image in this firmware image + * @ram_code_size: size of SDMA ram image + * @script_addrs: Stores the start address of the SDMA scripts * (in SDMA memory space) */ struct sdma_firmware_header { From 4a6b2e8a9a0216697405560f3a14b5f33812f444 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 24 Jul 2018 01:46:10 +0800 Subject: [PATCH 11/12] dmaengine: imx-sdma: add SDMA_BD_MAX_CNT to replace '0xffff' Add macro SDMA_BD_MAX_CNT to replace '0xffff'. Signed-off-by: Robin Gong Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 3b622d660c8b..e3d5e73a97c7 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -185,6 +185,7 @@ * Mode/Count of data node descriptors - IPCv2 */ struct sdma_mode_count { +#define SDMA_BD_MAX_CNT 0xffff u32 count : 16; /* size of the buffer pointed by this BD */ u32 status : 8; /* E,R,I,C,W,D status bits stored here */ u32 command : 8; /* command mostly used for channel 0 */ @@ -1344,9 +1345,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( count = sg_dma_len(sg); - if (count > 0xffff) { + if (count > SDMA_BD_MAX_CNT) { dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n", - channel, count, 0xffff); + channel, count, SDMA_BD_MAX_CNT); goto err_bd_out; } @@ -1421,9 +1422,9 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( sdmac->flags |= IMX_DMA_SG_LOOP; - if (period_len > 0xffff) { + if (period_len > SDMA_BD_MAX_CNT) { dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n", - channel, period_len, 0xffff); + channel, period_len, SDMA_BD_MAX_CNT); goto err_bd_out; } @@ -1970,7 +1971,7 @@ static int sdma_probe(struct platform_device *pdev) sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; sdma->dma_device.device_issue_pending = sdma_issue_pending; sdma->dma_device.dev->dma_parms = &sdma->dma_parms; - dma_set_max_seg_size(sdma->dma_device.dev, 65535); + dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT); platform_set_drvdata(pdev, sdma); From 0f06c02755889a9595a0883c8f4b75b2af188408 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 24 Jul 2018 01:46:11 +0800 Subject: [PATCH 12/12] dmaengine: imx-sdma: add memcpy interface Add MEMCPY capability for imx-sdma driver. Signed-off-by: Robin Gong Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 93 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index e3d5e73a97c7..b4ec2d20e661 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -342,6 +342,7 @@ struct sdma_desc { * @pc_from_device: script address for those device_2_memory * @pc_to_device: script address for those memory_2_device * @device_to_device: script address for those device_2_device + * @pc_to_pc: script address for those memory_2_memory * @flags: loop mode or not * @per_address: peripheral source or destination address in common case * destination address in p_2_p case @@ -367,6 +368,7 @@ struct sdma_channel { enum dma_slave_buswidth word_size; unsigned int pc_from_device, pc_to_device; unsigned int device_to_device; + unsigned int pc_to_pc; unsigned long flags; dma_addr_t per_address, per_address2; unsigned long event_mask[2]; @@ -869,14 +871,16 @@ static void sdma_get_pc(struct sdma_channel *sdmac, * These are needed once we start to support transfers between * two peripherals or memory-to-memory transfers */ - int per_2_per = 0; + int per_2_per = 0, emi_2_emi = 0; sdmac->pc_from_device = 0; sdmac->pc_to_device = 0; sdmac->device_to_device = 0; + sdmac->pc_to_pc = 0; switch (peripheral_type) { case IMX_DMATYPE_MEMORY: + emi_2_emi = sdma->script_addrs->ap_2_ap_addr; break; case IMX_DMATYPE_DSP: emi_2_per = sdma->script_addrs->bp_2_ap_addr; @@ -949,6 +953,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_from_device = per_2_emi; sdmac->pc_to_device = emi_2_per; sdmac->device_to_device = per_2_per; + sdmac->pc_to_pc = emi_2_emi; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -965,6 +970,8 @@ static int sdma_load_context(struct sdma_channel *sdmac) load_address = sdmac->pc_from_device; else if (sdmac->direction == DMA_DEV_TO_DEV) load_address = sdmac->device_to_device; + else if (sdmac->direction == DMA_MEM_TO_MEM) + load_address = sdmac->pc_to_pc; else load_address = sdmac->pc_to_device; @@ -1214,10 +1221,28 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); struct imx_dma_data *data = chan->private; + struct imx_dma_data mem_data; int prio, ret; - if (!data) - return -EINVAL; + /* + * MEMCPY may never setup chan->private by filter function such as + * dmatest, thus create 'struct imx_dma_data mem_data' for this case. + * Please note in any other slave case, you have to setup chan->private + * with 'struct imx_dma_data' in your own filter function if you want to + * request dma channel by dma_request_channel() rather than + * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear + * to warn you to correct your filter function. + */ + if (!data) { + dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n"); + mem_data.priority = 2; + mem_data.peripheral_type = IMX_DMATYPE_MEMORY; + mem_data.dma_request = 0; + mem_data.dma_request2 = 0; + data = &mem_data; + + sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY); + } switch (data->priority) { case DMA_PRIO_HIGH: @@ -1307,6 +1332,10 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, if (sdma_alloc_bd(desc)) goto err_desc_out; + /* No slave_config called in MEMCPY case, so do here */ + if (direction == DMA_MEM_TO_MEM) + sdma_config_ownership(sdmac, false, true, false); + if (sdma_load_context(sdmac)) goto err_desc_out; @@ -1318,6 +1347,62 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, return NULL; } +static struct dma_async_tx_descriptor *sdma_prep_memcpy( + struct dma_chan *chan, dma_addr_t dma_dst, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + struct sdma_channel *sdmac = to_sdma_chan(chan); + struct sdma_engine *sdma = sdmac->sdma; + int channel = sdmac->channel; + size_t count; + int i = 0, param; + struct sdma_buffer_descriptor *bd; + struct sdma_desc *desc; + + if (!chan || !len) + return NULL; + + dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n", + &dma_src, &dma_dst, len, channel); + + desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM, + len / SDMA_BD_MAX_CNT + 1); + if (!desc) + return NULL; + + do { + count = min_t(size_t, len, SDMA_BD_MAX_CNT); + bd = &desc->bd[i]; + bd->buffer_addr = dma_src; + bd->ext_buffer_addr = dma_dst; + bd->mode.count = count; + desc->chn_count += count; + bd->mode.command = 0; + + dma_src += count; + dma_dst += count; + len -= count; + i++; + + param = BD_DONE | BD_EXTD | BD_CONT; + /* last bd */ + if (!len) { + param |= BD_INTR; + param |= BD_LAST; + param &= ~BD_CONT; + } + + dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n", + i, count, bd->buffer_addr, + param & BD_WRAP ? "wrap" : "", + param & BD_INTR ? " intr" : ""); + + bd->mode.status = param; + } while (len); + + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); +} + static struct dma_async_tx_descriptor *sdma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, @@ -1903,6 +1988,7 @@ static int sdma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask); dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask); + dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask); INIT_LIST_HEAD(&sdma->dma_device.channels); /* Initialize channel parameters */ @@ -1969,6 +2055,7 @@ static int sdma_probe(struct platform_device *pdev) sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS; sdma->dma_device.directions = SDMA_DMA_DIRECTIONS; sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy; sdma->dma_device.device_issue_pending = sdma_issue_pending; sdma->dma_device.dev->dma_parms = &sdma->dma_parms; dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);