dmaengine: pl330: flush before wait, and add dev burst support.
Do DMAFLUSHP _before_ the first DMAWFP to ensure controller and peripheral are in agreement about dma request state before first transfer. Add support for burst transfers to/from peripherals. In the new scheme, the controller does as many burst transfers as it can then transfers the remaining dregs with either single transfers for peripherals, or with a reduced size burst for memory-to-memory transfers. Signed-off-by: Frank Mori Hess <fmh6jj@gmail.com> Tested-by: Frank Mori Hess <fmh6jj@gmail.com> Signed-off-by: Vinod Koul <vkoul@kernel.org>
This commit is contained in:
Frank Mori Hess
Vinod Koul
parent
60cc43fc88
commit
1d48745b19
|
|
@ -27,6 +27,7 @@
|
|||
#include <linux/of_dma.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/pm_runtime.h>
|
||||
#include <linux/bug.h>
|
||||
|
||||
#include "dmaengine.h"
|
||||
#define PL330_MAX_CHAN 8
|
||||
|
|
@ -1094,51 +1095,96 @@ static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
|
|||
return off;
|
||||
}
|
||||
|
||||
static inline int _ldst_devtomem(struct pl330_dmac *pl330, unsigned dry_run,
|
||||
u8 buf[], const struct _xfer_spec *pxs,
|
||||
int cyc)
|
||||
static u32 _emit_load(unsigned int dry_run, u8 buf[],
|
||||
enum pl330_cond cond, enum dma_transfer_direction direction,
|
||||
u8 peri)
|
||||
{
|
||||
int off = 0;
|
||||
enum pl330_cond cond;
|
||||
|
||||
if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
|
||||
cond = BURST;
|
||||
else
|
||||
cond = SINGLE;
|
||||
switch (direction) {
|
||||
case DMA_MEM_TO_MEM:
|
||||
/* fall through */
|
||||
case DMA_MEM_TO_DEV:
|
||||
off += _emit_LD(dry_run, &buf[off], cond);
|
||||
break;
|
||||
|
||||
while (cyc--) {
|
||||
off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
|
||||
off += _emit_LDP(dry_run, &buf[off], cond, pxs->desc->peri);
|
||||
off += _emit_ST(dry_run, &buf[off], ALWAYS);
|
||||
case DMA_DEV_TO_MEM:
|
||||
if (cond == ALWAYS) {
|
||||
off += _emit_LDP(dry_run, &buf[off], SINGLE,
|
||||
peri);
|
||||
off += _emit_LDP(dry_run, &buf[off], BURST,
|
||||
peri);
|
||||
} else {
|
||||
off += _emit_LDP(dry_run, &buf[off], cond,
|
||||
peri);
|
||||
}
|
||||
break;
|
||||
|
||||
if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
|
||||
off += _emit_FLUSHP(dry_run, &buf[off],
|
||||
pxs->desc->peri);
|
||||
default:
|
||||
/* this code should be unreachable */
|
||||
WARN_ON(1);
|
||||
break;
|
||||
}
|
||||
|
||||
return off;
|
||||
}
|
||||
|
||||
static inline int _ldst_memtodev(struct pl330_dmac *pl330,
|
||||
unsigned dry_run, u8 buf[],
|
||||
const struct _xfer_spec *pxs, int cyc)
|
||||
static inline u32 _emit_store(unsigned int dry_run, u8 buf[],
|
||||
enum pl330_cond cond, enum dma_transfer_direction direction,
|
||||
u8 peri)
|
||||
{
|
||||
int off = 0;
|
||||
|
||||
switch (direction) {
|
||||
case DMA_MEM_TO_MEM:
|
||||
/* fall through */
|
||||
case DMA_DEV_TO_MEM:
|
||||
off += _emit_ST(dry_run, &buf[off], cond);
|
||||
break;
|
||||
|
||||
case DMA_MEM_TO_DEV:
|
||||
if (cond == ALWAYS) {
|
||||
off += _emit_STP(dry_run, &buf[off], SINGLE,
|
||||
peri);
|
||||
off += _emit_STP(dry_run, &buf[off], BURST,
|
||||
peri);
|
||||
} else {
|
||||
off += _emit_STP(dry_run, &buf[off], cond,
|
||||
peri);
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
/* this code should be unreachable */
|
||||
WARN_ON(1);
|
||||
break;
|
||||
}
|
||||
|
||||
return off;
|
||||
}
|
||||
|
||||
static inline int _ldst_peripheral(struct pl330_dmac *pl330,
|
||||
unsigned dry_run, u8 buf[],
|
||||
const struct _xfer_spec *pxs, int cyc,
|
||||
enum pl330_cond cond)
|
||||
{
|
||||
int off = 0;
|
||||
enum pl330_cond cond;
|
||||
|
||||
if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
|
||||
cond = BURST;
|
||||
else
|
||||
cond = SINGLE;
|
||||
|
||||
/*
|
||||
* do FLUSHP at beginning to clear any stale dma requests before the
|
||||
* first WFP.
|
||||
*/
|
||||
if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
|
||||
off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
|
||||
while (cyc--) {
|
||||
off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
|
||||
off += _emit_LD(dry_run, &buf[off], ALWAYS);
|
||||
off += _emit_STP(dry_run, &buf[off], cond, pxs->desc->peri);
|
||||
|
||||
if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
|
||||
off += _emit_FLUSHP(dry_run, &buf[off],
|
||||
pxs->desc->peri);
|
||||
off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,
|
||||
pxs->desc->peri);
|
||||
off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,
|
||||
pxs->desc->peri);
|
||||
}
|
||||
|
||||
return off;
|
||||
|
|
@ -1148,19 +1194,65 @@ static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
|
|||
const struct _xfer_spec *pxs, int cyc)
|
||||
{
|
||||
int off = 0;
|
||||
enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE;
|
||||
|
||||
switch (pxs->desc->rqtype) {
|
||||
case DMA_MEM_TO_DEV:
|
||||
off += _ldst_memtodev(pl330, dry_run, &buf[off], pxs, cyc);
|
||||
break;
|
||||
/* fall through */
|
||||
case DMA_DEV_TO_MEM:
|
||||
off += _ldst_devtomem(pl330, dry_run, &buf[off], pxs, cyc);
|
||||
off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc,
|
||||
cond);
|
||||
break;
|
||||
|
||||
case DMA_MEM_TO_MEM:
|
||||
off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
|
||||
break;
|
||||
|
||||
default:
|
||||
off += 0x40000000; /* Scare off the Client */
|
||||
/* this code should be unreachable */
|
||||
WARN_ON(1);
|
||||
break;
|
||||
}
|
||||
|
||||
return off;
|
||||
}
|
||||
|
||||
/*
|
||||
* transfer dregs with single transfers to peripheral, or a reduced size burst
|
||||
* for mem-to-mem.
|
||||
*/
|
||||
static int _dregs(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[],
|
||||
const struct _xfer_spec *pxs, int transfer_length)
|
||||
{
|
||||
int off = 0;
|
||||
int dregs_ccr;
|
||||
|
||||
if (transfer_length == 0)
|
||||
return off;
|
||||
|
||||
switch (pxs->desc->rqtype) {
|
||||
case DMA_MEM_TO_DEV:
|
||||
/* fall through */
|
||||
case DMA_DEV_TO_MEM:
|
||||
off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs,
|
||||
transfer_length, SINGLE);
|
||||
break;
|
||||
|
||||
case DMA_MEM_TO_MEM:
|
||||
dregs_ccr = pxs->ccr;
|
||||
dregs_ccr &= ~((0xf << CC_SRCBRSTLEN_SHFT) |
|
||||
(0xf << CC_DSTBRSTLEN_SHFT));
|
||||
dregs_ccr |= (((transfer_length - 1) & 0xf) <<
|
||||
CC_SRCBRSTLEN_SHFT);
|
||||
dregs_ccr |= (((transfer_length - 1) & 0xf) <<
|
||||
CC_DSTBRSTLEN_SHFT);
|
||||
off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);
|
||||
off += _ldst_memtomem(dry_run, &buf[off], pxs, 1);
|
||||
break;
|
||||
|
||||
default:
|
||||
/* this code should be unreachable */
|
||||
WARN_ON(1);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -1256,6 +1348,8 @@ static inline int _setup_loops(struct pl330_dmac *pl330,
|
|||
struct pl330_xfer *x = &pxs->desc->px;
|
||||
u32 ccr = pxs->ccr;
|
||||
unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
|
||||
int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) /
|
||||
BRST_SIZE(ccr);
|
||||
int off = 0;
|
||||
|
||||
while (bursts) {
|
||||
|
|
@ -1263,6 +1357,7 @@ static inline int _setup_loops(struct pl330_dmac *pl330,
|
|||
off += _loop(pl330, dry_run, &buf[off], &c, pxs);
|
||||
bursts -= c;
|
||||
}
|
||||
off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs);
|
||||
|
||||
return off;
|
||||
}
|
||||
|
|
@ -1294,7 +1389,6 @@ static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
|
|||
struct _xfer_spec *pxs)
|
||||
{
|
||||
struct _pl330_req *req = &thrd->req[index];
|
||||
struct pl330_xfer *x;
|
||||
u8 *buf = req->mc_cpu;
|
||||
int off = 0;
|
||||
|
||||
|
|
@ -1303,11 +1397,6 @@ static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
|
|||
/* DMAMOV CCR, ccr */
|
||||
off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
|
||||
|
||||
x = &pxs->desc->px;
|
||||
/* Error if xfer length is not aligned at burst size */
|
||||
if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
|
||||
return -EINVAL;
|
||||
|
||||
off += _setup_xfer(pl330, dry_run, &buf[off], pxs);
|
||||
|
||||
/* DMASEV peripheral/event */
|
||||
|
|
@ -1365,6 +1454,20 @@ static int pl330_submit_req(struct pl330_thread *thrd,
|
|||
u32 ccr;
|
||||
int ret = 0;
|
||||
|
||||
switch (desc->rqtype) {
|
||||
case DMA_MEM_TO_DEV:
|
||||
break;
|
||||
|
||||
case DMA_DEV_TO_MEM:
|
||||
break;
|
||||
|
||||
case DMA_MEM_TO_MEM:
|
||||
break;
|
||||
|
||||
default:
|
||||
return -ENOTSUPP;
|
||||
}
|
||||
|
||||
if (pl330->state == DYING
|
||||
|| pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
|
||||
dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
|
||||
|
|
@ -2106,6 +2209,18 @@ static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,
|
|||
return true;
|
||||
}
|
||||
|
||||
static int fixup_burst_len(int max_burst_len, int quirks)
|
||||
{
|
||||
if (quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
|
||||
return 1;
|
||||
else if (max_burst_len > PL330_MAX_BURST)
|
||||
return PL330_MAX_BURST;
|
||||
else if (max_burst_len < 1)
|
||||
return 1;
|
||||
else
|
||||
return max_burst_len;
|
||||
}
|
||||
|
||||
static int pl330_config(struct dma_chan *chan,
|
||||
struct dma_slave_config *slave_config)
|
||||
{
|
||||
|
|
@ -2117,15 +2232,15 @@ static int pl330_config(struct dma_chan *chan,
|
|||
pch->fifo_addr = slave_config->dst_addr;
|
||||
if (slave_config->dst_addr_width)
|
||||
pch->burst_sz = __ffs(slave_config->dst_addr_width);
|
||||
if (slave_config->dst_maxburst)
|
||||
pch->burst_len = slave_config->dst_maxburst;
|
||||
pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,
|
||||
pch->dmac->quirks);
|
||||
} else if (slave_config->direction == DMA_DEV_TO_MEM) {
|
||||
if (slave_config->src_addr)
|
||||
pch->fifo_addr = slave_config->src_addr;
|
||||
if (slave_config->src_addr_width)
|
||||
pch->burst_sz = __ffs(slave_config->src_addr_width);
|
||||
if (slave_config->src_maxburst)
|
||||
pch->burst_len = slave_config->src_maxburst;
|
||||
pch->burst_len = fixup_burst_len(slave_config->src_maxburst,
|
||||
pch->dmac->quirks);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
@ -2519,14 +2634,8 @@ static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
|
|||
burst_len >>= desc->rqcfg.brst_size;
|
||||
|
||||
/* src/dst_burst_len can't be more than 16 */
|
||||
if (burst_len > 16)
|
||||
burst_len = 16;
|
||||
|
||||
while (burst_len > 1) {
|
||||
if (!(len % (burst_len << desc->rqcfg.brst_size)))
|
||||
break;
|
||||
burst_len--;
|
||||
}
|
||||
if (burst_len > PL330_MAX_BURST)
|
||||
burst_len = PL330_MAX_BURST;
|
||||
|
||||
return burst_len;
|
||||
}
|
||||
|
|
@ -2598,7 +2707,7 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
|
|||
|
||||
desc->rqtype = direction;
|
||||
desc->rqcfg.brst_size = pch->burst_sz;
|
||||
desc->rqcfg.brst_len = 1;
|
||||
desc->rqcfg.brst_len = pch->burst_len;
|
||||
desc->bytes_requested = period_len;
|
||||
fill_px(&desc->px, dst, src, period_len);
|
||||
|
||||
|
|
@ -2743,7 +2852,7 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
|
|||
}
|
||||
|
||||
desc->rqcfg.brst_size = pch->burst_sz;
|
||||
desc->rqcfg.brst_len = 1;
|
||||
desc->rqcfg.brst_len = pch->burst_len;
|
||||
desc->rqtype = direction;
|
||||
desc->bytes_requested = sg_dma_len(sg);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue