diff --git a/arch/arm/plat-nomadik/include/plat/ste_dma40.h b/arch/arm/plat-nomadik/include/plat/ste_dma40.h index 74b62f10d07f..4d6dd4c39b75 100644 --- a/arch/arm/plat-nomadik/include/plat/ste_dma40.h +++ b/arch/arm/plat-nomadik/include/plat/ste_dma40.h @@ -13,6 +13,14 @@ #include #include +/* + * Maxium size for a single dma descriptor + * Size is limited to 16 bits. + * Size is in the units of addr-widths (1,2,4,8 bytes) + * Larger transfers will be split up to multiple linked desc + */ +#define STEDMA40_MAX_SEG_SIZE 0xFFFF + /* dev types for memcpy */ #define STEDMA40_DEV_DST_MEMORY (-1) #define STEDMA40_DEV_SRC_MEMORY (-1) diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index b605cc9ac3a2..bebc678ed4fc 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -19,7 +19,7 @@ * this program; if not, write to the Free Software Foundation, Inc., 59 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. * - * The full GNU General Public License is iin this distribution in the + * The full GNU General Public License is in this distribution in the * file called COPYING. * * Documentation: ARM DDI 0196G == PL080 @@ -53,7 +53,23 @@ * * ASSUMES default (little) endianness for DMA transfers * - * Only DMAC flow control is implemented + * The PL08x has two flow control settings: + * - DMAC flow control: the transfer size defines the number of transfers + * which occur for the current LLI entry, and the DMAC raises TC at the + * end of every LLI entry. Observed behaviour shows the DMAC listening + * to both the BREQ and SREQ signals (contrary to documented), + * transferring data if either is active. The LBREQ and LSREQ signals + * are ignored. + * + * - Peripheral flow control: the transfer size is ignored (and should be + * zero). The data is transferred from the current LLI entry, until + * after the final transfer signalled by LBREQ or LSREQ. The DMAC + * will then move to the next LLI entry. + * + * Only the former works sanely with scatter lists, so we only implement + * the DMAC flow control method. However, peripherals which use the LBREQ + * and LSREQ signals (eg, MMCI) are unable to use this mode, which through + * these hardware restrictions prevents them from using scatter DMA. * * Global TODO: * - Break out common code from arch/arm/mach-s3c64xx and share @@ -61,50 +77,41 @@ #include #include #include -#include #include #include #include -#include #include +#include #include #include #include #include -#include -#include -#include -#include -#include #define DRIVER_NAME "pl08xdmac" /** * struct vendor_data - vendor-specific config parameters - * for PL08x derivates - * @name: the name of this specific variant + * for PL08x derivatives * @channels: the number of channels available in this variant * @dualmaster: whether this version supports dual AHB masters * or not. */ struct vendor_data { - char *name; u8 channels; bool dualmaster; }; /* * PL08X private data structures - * An LLI struct - see pl08x TRM - * Note that next uses bit[0] as a bus bit, - * start & end do not - their bus bit info - * is in cctl + * An LLI struct - see PL08x TRM. Note that next uses bit[0] as a bus bit, + * start & end do not - their bus bit info is in cctl. Also note that these + * are fixed 32-bit quantities. */ -struct lli { - dma_addr_t src; - dma_addr_t dst; - dma_addr_t next; +struct pl08x_lli { + u32 src; + u32 dst; + u32 lli; u32 cctl; }; @@ -119,6 +126,8 @@ struct lli { * @phy_chans: array of data for the physical channels * @pool: a pool for the LLI descriptors * @pool_ctr: counter of LLIs in the pool + * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI fetches + * @mem_buses: set to indicate memory transfers on AHB2. * @lock: a spinlock for this struct */ struct pl08x_driver_data { @@ -126,11 +135,13 @@ struct pl08x_driver_data { struct dma_device memcpy; void __iomem *base; struct amba_device *adev; - struct vendor_data *vd; + const struct vendor_data *vd; struct pl08x_platform_data *pd; struct pl08x_phy_chan *phy_chans; struct dma_pool *pool; int pool_ctr; + u8 lli_buses; + u8 mem_buses; spinlock_t lock; }; @@ -152,9 +163,9 @@ struct pl08x_driver_data { /* Size (bytes) of each LLI buffer allocated for one transfer */ # define PL08X_LLI_TSFR_SIZE 0x2000 -/* Maximimum times we call dma_pool_alloc on this pool without freeing */ +/* Maximum times we call dma_pool_alloc on this pool without freeing */ #define PL08X_MAX_ALLOCS 0x40 -#define MAX_NUM_TSFR_LLIS (PL08X_LLI_TSFR_SIZE/sizeof(struct lli)) +#define MAX_NUM_TSFR_LLIS (PL08X_LLI_TSFR_SIZE/sizeof(struct pl08x_lli)) #define PL08X_ALIGN 8 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) @@ -162,6 +173,11 @@ static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) return container_of(chan, struct pl08x_dma_chan, chan); } +static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct pl08x_txd, tx); +} + /* * Physical channel handling */ @@ -177,88 +193,47 @@ static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) /* * Set the initial DMA register values i.e. those for the first LLI - * The next lli pointer and the configuration interrupt bit have - * been set when the LLIs were constructed + * The next LLI pointer and the configuration interrupt bit have + * been set when the LLIs were constructed. Poke them into the hardware + * and start the transfer. */ -static void pl08x_set_cregs(struct pl08x_driver_data *pl08x, - struct pl08x_phy_chan *ch) +static void pl08x_start_txd(struct pl08x_dma_chan *plchan, + struct pl08x_txd *txd) { - /* Wait for channel inactive */ - while (pl08x_phy_channel_busy(ch)) - ; - - dev_vdbg(&pl08x->adev->dev, - "WRITE channel %d: csrc=%08x, cdst=%08x, " - "cctl=%08x, clli=%08x, ccfg=%08x\n", - ch->id, - ch->csrc, - ch->cdst, - ch->cctl, - ch->clli, - ch->ccfg); - - writel(ch->csrc, ch->base + PL080_CH_SRC_ADDR); - writel(ch->cdst, ch->base + PL080_CH_DST_ADDR); - writel(ch->clli, ch->base + PL080_CH_LLI); - writel(ch->cctl, ch->base + PL080_CH_CONTROL); - writel(ch->ccfg, ch->base + PL080_CH_CONFIG); -} - -static inline void pl08x_config_phychan_for_txd(struct pl08x_dma_chan *plchan) -{ - struct pl08x_channel_data *cd = plchan->cd; + struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_phy_chan *phychan = plchan->phychan; - struct pl08x_txd *txd = plchan->at; - - /* Copy the basic control register calculated at transfer config */ - phychan->csrc = txd->csrc; - phychan->cdst = txd->cdst; - phychan->clli = txd->clli; - phychan->cctl = txd->cctl; - - /* Assign the signal to the proper control registers */ - phychan->ccfg = cd->ccfg; - phychan->ccfg &= ~PL080_CONFIG_SRC_SEL_MASK; - phychan->ccfg &= ~PL080_CONFIG_DST_SEL_MASK; - /* If it wasn't set from AMBA, ignore it */ - if (txd->direction == DMA_TO_DEVICE) - /* Select signal as destination */ - phychan->ccfg |= - (phychan->signal << PL080_CONFIG_DST_SEL_SHIFT); - else if (txd->direction == DMA_FROM_DEVICE) - /* Select signal as source */ - phychan->ccfg |= - (phychan->signal << PL080_CONFIG_SRC_SEL_SHIFT); - /* Always enable error interrupts */ - phychan->ccfg |= PL080_CONFIG_ERR_IRQ_MASK; - /* Always enable terminal interrupts */ - phychan->ccfg |= PL080_CONFIG_TC_IRQ_MASK; -} - -/* - * Enable the DMA channel - * Assumes all other configuration bits have been set - * as desired before this code is called - */ -static void pl08x_enable_phy_chan(struct pl08x_driver_data *pl08x, - struct pl08x_phy_chan *ch) -{ + struct pl08x_lli *lli = &txd->llis_va[0]; u32 val; - /* - * Do not access config register until channel shows as disabled - */ - while (readl(pl08x->base + PL080_EN_CHAN) & (1 << ch->id)) - ; + plchan->at = txd; - /* - * Do not access config register until channel shows as inactive - */ - val = readl(ch->base + PL080_CH_CONFIG); + /* Wait for channel inactive */ + while (pl08x_phy_channel_busy(phychan)) + cpu_relax(); + + dev_vdbg(&pl08x->adev->dev, + "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " + "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", + phychan->id, lli->src, lli->dst, lli->lli, lli->cctl, + txd->ccfg); + + writel(lli->src, phychan->base + PL080_CH_SRC_ADDR); + writel(lli->dst, phychan->base + PL080_CH_DST_ADDR); + writel(lli->lli, phychan->base + PL080_CH_LLI); + writel(lli->cctl, phychan->base + PL080_CH_CONTROL); + writel(txd->ccfg, phychan->base + PL080_CH_CONFIG); + + /* Enable the DMA channel */ + /* Do not access config register until channel shows as disabled */ + while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id)) + cpu_relax(); + + /* Do not access config register until channel shows as inactive */ + val = readl(phychan->base + PL080_CH_CONFIG); while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE)) - val = readl(ch->base + PL080_CH_CONFIG); + val = readl(phychan->base + PL080_CH_CONFIG); - writel(val | PL080_CONFIG_ENABLE, ch->base + PL080_CH_CONFIG); + writel(val | PL080_CONFIG_ENABLE, phychan->base + PL080_CH_CONFIG); } /* @@ -282,7 +257,7 @@ static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) /* Wait for channel inactive */ while (pl08x_phy_channel_busy(ch)) - ; + cpu_relax(); } static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) @@ -333,54 +308,56 @@ static inline u32 get_bytes_in_cctl(u32 cctl) static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) { struct pl08x_phy_chan *ch; - struct pl08x_txd *txdi = NULL; struct pl08x_txd *txd; unsigned long flags; - u32 bytes = 0; + size_t bytes = 0; spin_lock_irqsave(&plchan->lock, flags); - ch = plchan->phychan; txd = plchan->at; /* - * Next follow the LLIs to get the number of pending bytes in the - * currently active transaction. + * Follow the LLIs to get the number of remaining + * bytes in the currently active transaction. */ if (ch && txd) { - struct lli *llis_va = txd->llis_va; - struct lli *llis_bus = (struct lli *) txd->llis_bus; - u32 clli = readl(ch->base + PL080_CH_LLI); + u32 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2; - /* First get the bytes in the current active LLI */ + /* First get the remaining bytes in the active transfer */ bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL)); if (clli) { - int i = 0; + struct pl08x_lli *llis_va = txd->llis_va; + dma_addr_t llis_bus = txd->llis_bus; + int index; - /* Forward to the LLI pointed to by clli */ - while ((clli != (u32) &(llis_bus[i])) && - (i < MAX_NUM_TSFR_LLIS)) - i++; + BUG_ON(clli < llis_bus || clli >= llis_bus + + sizeof(struct pl08x_lli) * MAX_NUM_TSFR_LLIS); + + /* + * Locate the next LLI - as this is an array, + * it's simple maths to find. + */ + index = (clli - llis_bus) / sizeof(struct pl08x_lli); + + for (; index < MAX_NUM_TSFR_LLIS; index++) { + bytes += get_bytes_in_cctl(llis_va[index].cctl); - while (clli) { - bytes += get_bytes_in_cctl(llis_va[i].cctl); /* - * A clli of 0x00000000 will terminate the - * LLI list + * A LLI pointer of 0 terminates the LLI list */ - clli = llis_va[i].next; - i++; + if (!llis_va[index].lli) + break; } } } /* Sum up all queued transactions */ - if (!list_empty(&plchan->desc_list)) { - list_for_each_entry(txdi, &plchan->desc_list, node) { + if (!list_empty(&plchan->pend_list)) { + struct pl08x_txd *txdi; + list_for_each_entry(txdi, &plchan->pend_list, node) { bytes += txdi->len; } - } spin_unlock_irqrestore(&plchan->lock, flags); @@ -465,11 +442,11 @@ static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded) } static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth, - u32 tsize) + size_t tsize) { u32 retbits = cctl; - /* Remove all src, dst and transfersize bits */ + /* Remove all src, dst and transfer size bits */ retbits &= ~PL080_CONTROL_DWIDTH_MASK; retbits &= ~PL080_CONTROL_SWIDTH_MASK; retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; @@ -509,38 +486,45 @@ static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth, return retbits; } +struct pl08x_lli_build_data { + struct pl08x_txd *txd; + struct pl08x_driver_data *pl08x; + struct pl08x_bus_data srcbus; + struct pl08x_bus_data dstbus; + size_t remainder; +}; + /* * Autoselect a master bus to use for the transfer * this prefers the destination bus if both available * if fixed address on one bus the other will be chosen */ -void pl08x_choose_master_bus(struct pl08x_bus_data *src_bus, - struct pl08x_bus_data *dst_bus, struct pl08x_bus_data **mbus, - struct pl08x_bus_data **sbus, u32 cctl) +static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd, + struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl) { if (!(cctl & PL080_CONTROL_DST_INCR)) { - *mbus = src_bus; - *sbus = dst_bus; + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; } else if (!(cctl & PL080_CONTROL_SRC_INCR)) { - *mbus = dst_bus; - *sbus = src_bus; + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; } else { - if (dst_bus->buswidth == 4) { - *mbus = dst_bus; - *sbus = src_bus; - } else if (src_bus->buswidth == 4) { - *mbus = src_bus; - *sbus = dst_bus; - } else if (dst_bus->buswidth == 2) { - *mbus = dst_bus; - *sbus = src_bus; - } else if (src_bus->buswidth == 2) { - *mbus = src_bus; - *sbus = dst_bus; + if (bd->dstbus.buswidth == 4) { + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; + } else if (bd->srcbus.buswidth == 4) { + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; + } else if (bd->dstbus.buswidth == 2) { + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; + } else if (bd->srcbus.buswidth == 2) { + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; } else { - /* src_bus->buswidth == 1 */ - *mbus = dst_bus; - *sbus = src_bus; + /* bd->srcbus.buswidth == 1 */ + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; } } } @@ -549,55 +533,41 @@ void pl08x_choose_master_bus(struct pl08x_bus_data *src_bus, * Fills in one LLI for a certain transfer descriptor * and advance the counter */ -int pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, - struct pl08x_txd *txd, int num_llis, int len, - u32 cctl, u32 *remainder) +static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd, + int num_llis, int len, u32 cctl) { - struct lli *llis_va = txd->llis_va; - struct lli *llis_bus = (struct lli *) txd->llis_bus; + struct pl08x_lli *llis_va = bd->txd->llis_va; + dma_addr_t llis_bus = bd->txd->llis_bus; BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); - llis_va[num_llis].cctl = cctl; - llis_va[num_llis].src = txd->srcbus.addr; - llis_va[num_llis].dst = txd->dstbus.addr; - - /* - * On versions with dual masters, you can optionally AND on - * PL080_LLI_LM_AHB2 to the LLI to tell the hardware to read - * in new LLIs with that controller, but we always try to - * choose AHB1 to point into memory. The idea is to have AHB2 - * fixed on the peripheral and AHB1 messing around in the - * memory. So we don't manipulate this bit currently. - */ - - llis_va[num_llis].next = - (dma_addr_t)((u32) &(llis_bus[num_llis + 1])); + llis_va[num_llis].cctl = cctl; + llis_va[num_llis].src = bd->srcbus.addr; + llis_va[num_llis].dst = bd->dstbus.addr; + llis_va[num_llis].lli = llis_bus + (num_llis + 1) * sizeof(struct pl08x_lli); + if (bd->pl08x->lli_buses & PL08X_AHB2) + llis_va[num_llis].lli |= PL080_LLI_LM_AHB2; if (cctl & PL080_CONTROL_SRC_INCR) - txd->srcbus.addr += len; + bd->srcbus.addr += len; if (cctl & PL080_CONTROL_DST_INCR) - txd->dstbus.addr += len; + bd->dstbus.addr += len; - *remainder -= len; + BUG_ON(bd->remainder < len); - return num_llis + 1; + bd->remainder -= len; } /* - * Return number of bytes to fill to boundary, or len + * Return number of bytes to fill to boundary, or len. + * This calculation works for any value of addr. */ -static inline u32 pl08x_pre_boundary(u32 addr, u32 len) +static inline size_t pl08x_pre_boundary(u32 addr, size_t len) { - u32 boundary; + size_t boundary_len = PL08X_BOUNDARY_SIZE - + (addr & (PL08X_BOUNDARY_SIZE - 1)); - boundary = ((addr >> PL08X_BOUNDARY_SHIFT) + 1) - << PL08X_BOUNDARY_SHIFT; - - if (boundary < addr + len) - return boundary - addr; - else - return len; + return min(boundary_len, len); } /* @@ -608,20 +578,13 @@ static inline u32 pl08x_pre_boundary(u32 addr, u32 len) static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, struct pl08x_txd *txd) { - struct pl08x_channel_data *cd = txd->cd; struct pl08x_bus_data *mbus, *sbus; - u32 remainder; + struct pl08x_lli_build_data bd; int num_llis = 0; u32 cctl; - int max_bytes_per_lli; - int total_bytes = 0; - struct lli *llis_va; - struct lli *llis_bus; - - if (!txd) { - dev_err(&pl08x->adev->dev, "%s no descriptor\n", __func__); - return 0; - } + size_t max_bytes_per_lli; + size_t total_bytes = 0; + struct pl08x_lli *llis_va; txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); @@ -632,101 +595,67 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, pl08x->pool_ctr++; - /* - * Initialize bus values for this transfer - * from the passed optimal values - */ - if (!cd) { - dev_err(&pl08x->adev->dev, "%s no channel data\n", __func__); - return 0; - } + /* Get the default CCTL */ + cctl = txd->cctl; - /* Get the default CCTL from the platform data */ - cctl = cd->cctl; - - /* - * On the PL080 we have two bus masters and we - * should select one for source and one for - * destination. We try to use AHB2 for the - * bus which does not increment (typically the - * peripheral) else we just choose something. - */ - cctl &= ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2); - if (pl08x->vd->dualmaster) { - if (cctl & PL080_CONTROL_SRC_INCR) - /* Source increments, use AHB2 for destination */ - cctl |= PL080_CONTROL_DST_AHB2; - else if (cctl & PL080_CONTROL_DST_INCR) - /* Destination increments, use AHB2 for source */ - cctl |= PL080_CONTROL_SRC_AHB2; - else - /* Just pick something, source AHB1 dest AHB2 */ - cctl |= PL080_CONTROL_DST_AHB2; - } + bd.txd = txd; + bd.pl08x = pl08x; + bd.srcbus.addr = txd->src_addr; + bd.dstbus.addr = txd->dst_addr; /* Find maximum width of the source bus */ - txd->srcbus.maxwidth = + bd.srcbus.maxwidth = pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >> PL080_CONTROL_SWIDTH_SHIFT); /* Find maximum width of the destination bus */ - txd->dstbus.maxwidth = + bd.dstbus.maxwidth = pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >> PL080_CONTROL_DWIDTH_SHIFT); /* Set up the bus widths to the maximum */ - txd->srcbus.buswidth = txd->srcbus.maxwidth; - txd->dstbus.buswidth = txd->dstbus.maxwidth; + bd.srcbus.buswidth = bd.srcbus.maxwidth; + bd.dstbus.buswidth = bd.dstbus.maxwidth; dev_vdbg(&pl08x->adev->dev, "%s source bus is %d bytes wide, dest bus is %d bytes wide\n", - __func__, txd->srcbus.buswidth, txd->dstbus.buswidth); + __func__, bd.srcbus.buswidth, bd.dstbus.buswidth); /* * Bytes transferred == tsize * MIN(buswidths), not max(buswidths) */ - max_bytes_per_lli = min(txd->srcbus.buswidth, txd->dstbus.buswidth) * + max_bytes_per_lli = min(bd.srcbus.buswidth, bd.dstbus.buswidth) * PL080_CONTROL_TRANSFER_SIZE_MASK; dev_vdbg(&pl08x->adev->dev, - "%s max bytes per lli = %d\n", + "%s max bytes per lli = %zu\n", __func__, max_bytes_per_lli); /* We need to count this down to zero */ - remainder = txd->len; + bd.remainder = txd->len; dev_vdbg(&pl08x->adev->dev, - "%s remainder = %d\n", - __func__, remainder); + "%s remainder = %zu\n", + __func__, bd.remainder); /* * Choose bus to align to * - prefers destination bus if both available * - if fixed address on one bus chooses other - * - modifies cctl to choose an apropriate master - */ - pl08x_choose_master_bus(&txd->srcbus, &txd->dstbus, - &mbus, &sbus, cctl); - - - /* - * The lowest bit of the LLI register - * is also used to indicate which master to - * use for reading the LLIs. + * - modifies cctl to choose an appropriate master */ + pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl); if (txd->len < mbus->buswidth) { /* * Less than a bus width available * - send as single bytes */ - while (remainder) { + while (bd.remainder) { dev_vdbg(&pl08x->adev->dev, "%s single byte LLIs for a transfer of " - "less than a bus width (remain %08x)\n", - __func__, remainder); + "less than a bus width (remain 0x%08x)\n", + __func__, bd.remainder); cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - num_llis = - pl08x_fill_lli_for_desc(pl08x, txd, num_llis, 1, - cctl, &remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); total_bytes++; } } else { @@ -737,11 +666,10 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, while ((mbus->addr) % (mbus->buswidth)) { dev_vdbg(&pl08x->adev->dev, "%s adjustment lli for less than bus width " - "(remain %08x)\n", - __func__, remainder); + "(remain 0x%08x)\n", + __func__, bd.remainder); cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - num_llis = pl08x_fill_lli_for_desc - (pl08x, txd, num_llis, 1, cctl, &remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); total_bytes++; } @@ -761,53 +689,43 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, * Make largest possible LLIs until less than one bus * width left */ - while (remainder > (mbus->buswidth - 1)) { - int lli_len, target_len; - int tsize; - int odd_bytes; + while (bd.remainder > (mbus->buswidth - 1)) { + size_t lli_len, target_len, tsize, odd_bytes; /* * If enough left try to send max possible, * otherwise try to send the remainder */ - target_len = remainder; - if (remainder > max_bytes_per_lli) - target_len = max_bytes_per_lli; + target_len = min(bd.remainder, max_bytes_per_lli); /* - * Set bus lengths for incrementing busses - * to number of bytes which fill to next memory - * boundary + * Set bus lengths for incrementing buses to the + * number of bytes which fill to next memory boundary, + * limiting on the target length calculated above. */ if (cctl & PL080_CONTROL_SRC_INCR) - txd->srcbus.fill_bytes = - pl08x_pre_boundary( - txd->srcbus.addr, - remainder); + bd.srcbus.fill_bytes = + pl08x_pre_boundary(bd.srcbus.addr, + target_len); else - txd->srcbus.fill_bytes = - max_bytes_per_lli; + bd.srcbus.fill_bytes = target_len; if (cctl & PL080_CONTROL_DST_INCR) - txd->dstbus.fill_bytes = - pl08x_pre_boundary( - txd->dstbus.addr, - remainder); + bd.dstbus.fill_bytes = + pl08x_pre_boundary(bd.dstbus.addr, + target_len); else - txd->dstbus.fill_bytes = - max_bytes_per_lli; + bd.dstbus.fill_bytes = target_len; - /* - * Find the nearest - */ - lli_len = min(txd->srcbus.fill_bytes, - txd->dstbus.fill_bytes); + /* Find the nearest */ + lli_len = min(bd.srcbus.fill_bytes, + bd.dstbus.fill_bytes); - BUG_ON(lli_len > remainder); + BUG_ON(lli_len > bd.remainder); if (lli_len <= 0) { dev_err(&pl08x->adev->dev, - "%s lli_len is %d, <= 0\n", + "%s lli_len is %zu, <= 0\n", __func__, lli_len); return 0; } @@ -826,7 +744,7 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, /* * So now we know how many bytes to transfer * to get to the nearest boundary - * The next lli will past the boundary + * The next LLI will past the boundary * - however we may be working to a boundary * on the slave bus * We need to ensure the master stays aligned @@ -855,21 +773,20 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, if (target_len != lli_len) { dev_vdbg(&pl08x->adev->dev, - "%s can't send what we want. Desired %08x, lli of %08x bytes in txd of %08x\n", + "%s can't send what we want. Desired 0x%08zx, lli of 0x%08zx bytes in txd of 0x%08zx\n", __func__, target_len, lli_len, txd->len); } cctl = pl08x_cctl_bits(cctl, - txd->srcbus.buswidth, - txd->dstbus.buswidth, + bd.srcbus.buswidth, + bd.dstbus.buswidth, tsize); dev_vdbg(&pl08x->adev->dev, - "%s fill lli with single lli chunk of size %08x (remainder %08x)\n", - __func__, lli_len, remainder); - num_llis = pl08x_fill_lli_for_desc(pl08x, txd, - num_llis, lli_len, cctl, - &remainder); + "%s fill lli with single lli chunk of size 0x%08zx (remainder 0x%08zx)\n", + __func__, lli_len, bd.remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, + lli_len, cctl); total_bytes += lli_len; } @@ -881,15 +798,13 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, */ int j; for (j = 0; (j < mbus->buswidth) - && (remainder); j++) { + && (bd.remainder); j++) { cctl = pl08x_cctl_bits(cctl, 1, 1, 1); dev_vdbg(&pl08x->adev->dev, - "%s align with boundardy, single byte (remain %08x)\n", - __func__, remainder); - num_llis = - pl08x_fill_lli_for_desc(pl08x, - txd, num_llis, 1, - cctl, &remainder); + "%s align with boundary, single byte (remain 0x%08zx)\n", + __func__, bd.remainder); + pl08x_fill_lli_for_desc(&bd, + num_llis++, 1, cctl); total_bytes++; } } @@ -898,25 +813,18 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, /* * Send any odd bytes */ - if (remainder < 0) { - dev_err(&pl08x->adev->dev, "%s remainder not fitted 0x%08x bytes\n", - __func__, remainder); - return 0; - } - - while (remainder) { + while (bd.remainder) { cctl = pl08x_cctl_bits(cctl, 1, 1, 1); dev_vdbg(&pl08x->adev->dev, - "%s align with boundardy, single odd byte (remain %d)\n", - __func__, remainder); - num_llis = pl08x_fill_lli_for_desc(pl08x, txd, num_llis, - 1, cctl, &remainder); + "%s align with boundary, single odd byte (remain %zu)\n", + __func__, bd.remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); total_bytes++; } } if (total_bytes != txd->len) { dev_err(&pl08x->adev->dev, - "%s size of encoded lli:s don't match total txd, transferred 0x%08x from size 0x%08x\n", + "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", __func__, total_bytes, txd->len); return 0; } @@ -927,41 +835,16 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, __func__, (u32) MAX_NUM_TSFR_LLIS); return 0; } - /* - * Decide whether this is a loop or a terminated transfer - */ + llis_va = txd->llis_va; - llis_bus = (struct lli *) txd->llis_bus; - - if (cd->circular_buffer) { - /* - * Loop the circular buffer so that the next element - * points back to the beginning of the LLI. - */ - llis_va[num_llis - 1].next = - (dma_addr_t)((unsigned int)&(llis_bus[0])); - } else { - /* - * On non-circular buffers, the final LLI terminates - * the LLI. - */ - llis_va[num_llis - 1].next = 0; - /* - * The final LLI element shall also fire an interrupt - */ - llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN; - } - - /* Now store the channel register values */ - txd->csrc = llis_va[0].src; - txd->cdst = llis_va[0].dst; - if (num_llis > 1) - txd->clli = llis_va[0].next; - else - txd->clli = 0; - - txd->cctl = llis_va[0].cctl; - /* ccfg will be set at physical channel allocation time */ + /* + * The final LLI terminates the LLI. + */ + llis_va[num_llis - 1].lli = 0; + /* + * The final LLI element shall also fire an interrupt + */ + llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN; #ifdef VERBOSE_DEBUG { @@ -969,13 +852,13 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, for (i = 0; i < num_llis; i++) { dev_vdbg(&pl08x->adev->dev, - "lli %d @%p: csrc=%08x, cdst=%08x, cctl=%08x, clli=%08x\n", + "lli %d @%p: csrc=0x%08x, cdst=0x%08x, cctl=0x%08x, clli=0x%08x\n", i, &llis_va[i], llis_va[i].src, llis_va[i].dst, llis_va[i].cctl, - llis_va[i].next + llis_va[i].lli ); } } @@ -988,14 +871,8 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, static void pl08x_free_txd(struct pl08x_driver_data *pl08x, struct pl08x_txd *txd) { - if (!txd) - dev_err(&pl08x->adev->dev, - "%s no descriptor to free\n", - __func__); - /* Free the LLI */ - dma_pool_free(pl08x->pool, txd->llis_va, - txd->llis_bus); + dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); pl08x->pool_ctr--; @@ -1008,9 +885,9 @@ static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, struct pl08x_txd *txdi = NULL; struct pl08x_txd *next; - if (!list_empty(&plchan->desc_list)) { + if (!list_empty(&plchan->pend_list)) { list_for_each_entry_safe(txdi, - next, &plchan->desc_list, node) { + next, &plchan->pend_list, node) { list_del(&txdi->node); pl08x_free_txd(pl08x, txdi); } @@ -1069,6 +946,12 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan, return -EBUSY; } ch->signal = ret; + + /* Assign the flow control signal to this channel */ + if (txd->direction == DMA_TO_DEVICE) + txd->ccfg |= ch->signal << PL080_CONFIG_DST_SEL_SHIFT; + else if (txd->direction == DMA_FROM_DEVICE) + txd->ccfg |= ch->signal << PL080_CONFIG_SRC_SEL_SHIFT; } dev_dbg(&pl08x->adev->dev, "allocated physical channel %d and signal %d for xfer on %s\n", @@ -1076,19 +959,54 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan, ch->signal, plchan->name); + plchan->phychan_hold++; plchan->phychan = ch; return 0; } +static void release_phy_channel(struct pl08x_dma_chan *plchan) +{ + struct pl08x_driver_data *pl08x = plchan->host; + + if ((plchan->phychan->signal >= 0) && pl08x->pd->put_signal) { + pl08x->pd->put_signal(plchan); + plchan->phychan->signal = -1; + } + pl08x_put_phy_channel(pl08x, plchan->phychan); + plchan->phychan = NULL; +} + static dma_cookie_t pl08x_tx_submit(struct dma_async_tx_descriptor *tx) { struct pl08x_dma_chan *plchan = to_pl08x_chan(tx->chan); + struct pl08x_txd *txd = to_pl08x_txd(tx); + unsigned long flags; - atomic_inc(&plchan->last_issued); - tx->cookie = atomic_read(&plchan->last_issued); - /* This unlock follows the lock in the prep() function */ - spin_unlock_irqrestore(&plchan->lock, plchan->lockflags); + spin_lock_irqsave(&plchan->lock, flags); + + plchan->chan.cookie += 1; + if (plchan->chan.cookie < 0) + plchan->chan.cookie = 1; + tx->cookie = plchan->chan.cookie; + + /* Put this onto the pending list */ + list_add_tail(&txd->node, &plchan->pend_list); + + /* + * If there was no physical channel available for this memcpy, + * stack the request up and indicate that the channel is waiting + * for a free physical channel. + */ + if (!plchan->slave && !plchan->phychan) { + /* Do this memcpy whenever there is a channel ready */ + plchan->state = PL08X_CHAN_WAITING; + plchan->waiting = txd; + } else { + plchan->phychan_hold--; + } + + spin_unlock_irqrestore(&plchan->lock, flags); return tx->cookie; } @@ -1118,7 +1036,7 @@ pl08x_dma_tx_status(struct dma_chan *chan, enum dma_status ret; u32 bytesleft = 0; - last_used = atomic_read(&plchan->last_issued); + last_used = plchan->chan.cookie; last_complete = plchan->lc; ret = dma_async_is_complete(cookie, last_complete, last_used); @@ -1134,7 +1052,7 @@ pl08x_dma_tx_status(struct dma_chan *chan, /* * This cookie not complete yet */ - last_used = atomic_read(&plchan->last_issued); + last_used = plchan->chan.cookie; last_complete = plchan->lc; /* Get number of bytes left in the active transactions and queue */ @@ -1199,37 +1117,35 @@ static const struct burst_table burst_sizes[] = { }, }; -static void dma_set_runtime_config(struct dma_chan *chan, - struct dma_slave_config *config) +static int dma_set_runtime_config(struct dma_chan *chan, + struct dma_slave_config *config) { struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_channel_data *cd = plchan->cd; enum dma_slave_buswidth addr_width; + dma_addr_t addr; u32 maxburst; u32 cctl = 0; - /* Mask out all except src and dst channel */ - u32 ccfg = cd->ccfg & 0x000003DEU; - int i = 0; + int i; + + if (!plchan->slave) + return -EINVAL; /* Transfer direction */ plchan->runtime_direction = config->direction; if (config->direction == DMA_TO_DEVICE) { - plchan->runtime_addr = config->dst_addr; - cctl |= PL080_CONTROL_SRC_INCR; - ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT; + addr = config->dst_addr; addr_width = config->dst_addr_width; maxburst = config->dst_maxburst; } else if (config->direction == DMA_FROM_DEVICE) { - plchan->runtime_addr = config->src_addr; - cctl |= PL080_CONTROL_DST_INCR; - ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; + addr = config->src_addr; addr_width = config->src_addr_width; maxburst = config->src_maxburst; } else { dev_err(&pl08x->adev->dev, "bad runtime_config: alien transfer direction\n"); - return; + return -EINVAL; } switch (addr_width) { @@ -1248,42 +1164,40 @@ static void dma_set_runtime_config(struct dma_chan *chan, default: dev_err(&pl08x->adev->dev, "bad runtime_config: alien address width\n"); - return; + return -EINVAL; } /* * Now decide on a maxburst: - * If this channel will only request single transfers, set - * this down to ONE element. + * If this channel will only request single transfers, set this + * down to ONE element. Also select one element if no maxburst + * is specified. */ - if (plchan->cd->single) { + if (plchan->cd->single || maxburst == 0) { cctl |= (PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT) | (PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT); } else { - while (i < ARRAY_SIZE(burst_sizes)) { + for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) if (burst_sizes[i].burstwords <= maxburst) break; - i++; - } cctl |= burst_sizes[i].reg; } - /* Access the cell in privileged mode, non-bufferable, non-cacheable */ - cctl &= ~PL080_CONTROL_PROT_MASK; - cctl |= PL080_CONTROL_PROT_SYS; + plchan->runtime_addr = addr; /* Modify the default channel data to fit PrimeCell request */ cd->cctl = cctl; - cd->ccfg = ccfg; dev_dbg(&pl08x->adev->dev, "configured channel %s (%s) for %s, data width %d, " - "maxburst %d words, LE, CCTL=%08x, CCFG=%08x\n", + "maxburst %d words, LE, CCTL=0x%08x\n", dma_chan_name(chan), plchan->name, (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX", addr_width, maxburst, - cctl, ccfg); + cctl); + + return 0; } /* @@ -1293,35 +1207,26 @@ static void dma_set_runtime_config(struct dma_chan *chan, static void pl08x_issue_pending(struct dma_chan *chan) { struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); - struct pl08x_driver_data *pl08x = plchan->host; unsigned long flags; spin_lock_irqsave(&plchan->lock, flags); - /* Something is already active */ - if (plchan->at) { - spin_unlock_irqrestore(&plchan->lock, flags); - return; + /* Something is already active, or we're waiting for a channel... */ + if (plchan->at || plchan->state == PL08X_CHAN_WAITING) { + spin_unlock_irqrestore(&plchan->lock, flags); + return; } - /* Didn't get a physical channel so waiting for it ... */ - if (plchan->state == PL08X_CHAN_WAITING) - return; - /* Take the first element in the queue and execute it */ - if (!list_empty(&plchan->desc_list)) { + if (!list_empty(&plchan->pend_list)) { struct pl08x_txd *next; - next = list_first_entry(&plchan->desc_list, + next = list_first_entry(&plchan->pend_list, struct pl08x_txd, node); list_del(&next->node); - plchan->at = next; plchan->state = PL08X_CHAN_RUNNING; - /* Configure the physical channel for the active txd */ - pl08x_config_phychan_for_txd(plchan); - pl08x_set_cregs(pl08x, plchan->phychan); - pl08x_enable_phy_chan(pl08x, plchan->phychan); + pl08x_start_txd(plchan, next); } spin_unlock_irqrestore(&plchan->lock, flags); @@ -1330,30 +1235,17 @@ static void pl08x_issue_pending(struct dma_chan *chan) static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, struct pl08x_txd *txd) { - int num_llis; struct pl08x_driver_data *pl08x = plchan->host; - int ret; + unsigned long flags; + int num_llis, ret; num_llis = pl08x_fill_llis_for_desc(pl08x, txd); - - if (!num_llis) + if (!num_llis) { + kfree(txd); return -EINVAL; + } - spin_lock_irqsave(&plchan->lock, plchan->lockflags); - - /* - * If this device is not using a circular buffer then - * queue this new descriptor for transfer. - * The descriptor for a circular buffer continues - * to be used until the channel is freed. - */ - if (txd->cd->circular_buffer) - dev_err(&pl08x->adev->dev, - "%s attempting to queue a circular buffer\n", - __func__); - else - list_add_tail(&txd->node, - &plchan->desc_list); + spin_lock_irqsave(&plchan->lock, flags); /* * See if we already have a physical channel allocated, @@ -1362,24 +1254,23 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, ret = prep_phy_channel(plchan, txd); if (ret) { /* - * No physical channel available, we will - * stack up the memcpy channels until there is a channel - * available to handle it whereas slave transfers may - * have been denied due to platform channel muxing restrictions - * and since there is no guarantee that this will ever be - * resolved, and since the signal must be aquired AFTER - * aquiring the physical channel, we will let them be NACK:ed - * with -EBUSY here. The drivers can alway retry the prep() - * call if they are eager on doing this using DMA. + * No physical channel was available. + * + * memcpy transfers can be sorted out at submission time. + * + * Slave transfers may have been denied due to platform + * channel muxing restrictions. Since there is no guarantee + * that this will ever be resolved, and the signal must be + * acquired AFTER acquiring the physical channel, we will let + * them be NACK:ed with -EBUSY here. The drivers can retry + * the prep() call if they are eager on doing this using DMA. */ if (plchan->slave) { pl08x_free_txd_list(pl08x, plchan); - spin_unlock_irqrestore(&plchan->lock, plchan->lockflags); + pl08x_free_txd(pl08x, txd); + spin_unlock_irqrestore(&plchan->lock, flags); return -EBUSY; } - /* Do this memcpy whenever there is a channel ready */ - plchan->state = PL08X_CHAN_WAITING; - plchan->waiting = txd; } else /* * Else we're all set, paused and ready to roll, @@ -1391,15 +1282,46 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, if (plchan->state == PL08X_CHAN_IDLE) plchan->state = PL08X_CHAN_PAUSED; - /* - * Notice that we leave plchan->lock locked on purpose: - * it will be unlocked in the subsequent tx_submit() - * call. This is a consequence of the current API. - */ + spin_unlock_irqrestore(&plchan->lock, flags); return 0; } +/* + * Given the source and destination available bus masks, select which + * will be routed to each port. We try to have source and destination + * on separate ports, but always respect the allowable settings. + */ +static u32 pl08x_select_bus(struct pl08x_driver_data *pl08x, u8 src, u8 dst) +{ + u32 cctl = 0; + + if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) + cctl |= PL080_CONTROL_DST_AHB2; + if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) + cctl |= PL080_CONTROL_SRC_AHB2; + + return cctl; +} + +static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan, + unsigned long flags) +{ + struct pl08x_txd *txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); + + if (txd) { + dma_async_tx_descriptor_init(&txd->tx, &plchan->chan); + txd->tx.flags = flags; + txd->tx.tx_submit = pl08x_tx_submit; + INIT_LIST_HEAD(&txd->node); + + /* Always enable error and terminal interrupts */ + txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | + PL080_CONFIG_TC_IRQ_MASK; + } + return txd; +} + /* * Initialize a descriptor to be used by memcpy submit */ @@ -1412,40 +1334,38 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( struct pl08x_txd *txd; int ret; - txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); + txd = pl08x_get_txd(plchan, flags); if (!txd) { dev_err(&pl08x->adev->dev, "%s no memory for descriptor\n", __func__); return NULL; } - dma_async_tx_descriptor_init(&txd->tx, chan); txd->direction = DMA_NONE; - txd->srcbus.addr = src; - txd->dstbus.addr = dest; - - /* Set platform data for m2m */ - txd->cd = &pl08x->pd->memcpy_channel; - /* Both to be incremented or the code will break */ - txd->cd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; - txd->tx.tx_submit = pl08x_tx_submit; - txd->tx.callback = NULL; - txd->tx.callback_param = NULL; + txd->src_addr = src; + txd->dst_addr = dest; txd->len = len; - INIT_LIST_HEAD(&txd->node); + /* Set platform data for m2m */ + txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; + txd->cctl = pl08x->pd->memcpy_channel.cctl & + ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2); + + /* Both to be incremented or the code will break */ + txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; + + if (pl08x->vd->dualmaster) + txd->cctl |= pl08x_select_bus(pl08x, + pl08x->mem_buses, pl08x->mem_buses); + ret = pl08x_prep_channel_resources(plchan, txd); if (ret) return NULL; - /* - * NB: the channel lock is held at this point so tx_submit() - * must be called in direct succession. - */ return &txd->tx; } -struct dma_async_tx_descriptor *pl08x_prep_slave_sg( +static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_data_direction direction, unsigned long flags) @@ -1453,6 +1373,7 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg( struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_txd *txd; + u8 src_buses, dst_buses; int ret; /* @@ -1467,14 +1388,12 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg( dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", __func__, sgl->length, plchan->name); - txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); + txd = pl08x_get_txd(plchan, flags); if (!txd) { dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); return NULL; } - dma_async_tx_descriptor_init(&txd->tx, chan); - if (direction != plchan->runtime_direction) dev_err(&pl08x->adev->dev, "%s DMA setup does not match " "the direction configured for the PrimeCell\n", @@ -1486,37 +1405,47 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg( * channel target address dynamically at runtime. */ txd->direction = direction; + txd->len = sgl->length; + + txd->cctl = plchan->cd->cctl & + ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | + PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | + PL080_CONTROL_PROT_MASK); + + /* Access the cell in privileged mode, non-bufferable, non-cacheable */ + txd->cctl |= PL080_CONTROL_PROT_SYS; + if (direction == DMA_TO_DEVICE) { - txd->srcbus.addr = sgl->dma_address; + txd->ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT; + txd->cctl |= PL080_CONTROL_SRC_INCR; + txd->src_addr = sgl->dma_address; if (plchan->runtime_addr) - txd->dstbus.addr = plchan->runtime_addr; + txd->dst_addr = plchan->runtime_addr; else - txd->dstbus.addr = plchan->cd->addr; + txd->dst_addr = plchan->cd->addr; + src_buses = pl08x->mem_buses; + dst_buses = plchan->cd->periph_buses; } else if (direction == DMA_FROM_DEVICE) { + txd->ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; + txd->cctl |= PL080_CONTROL_DST_INCR; if (plchan->runtime_addr) - txd->srcbus.addr = plchan->runtime_addr; + txd->src_addr = plchan->runtime_addr; else - txd->srcbus.addr = plchan->cd->addr; - txd->dstbus.addr = sgl->dma_address; + txd->src_addr = plchan->cd->addr; + txd->dst_addr = sgl->dma_address; + src_buses = plchan->cd->periph_buses; + dst_buses = pl08x->mem_buses; } else { dev_err(&pl08x->adev->dev, "%s direction unsupported\n", __func__); return NULL; } - txd->cd = plchan->cd; - txd->tx.tx_submit = pl08x_tx_submit; - txd->tx.callback = NULL; - txd->tx.callback_param = NULL; - txd->len = sgl->length; - INIT_LIST_HEAD(&txd->node); + + txd->cctl |= pl08x_select_bus(pl08x, src_buses, dst_buses); ret = pl08x_prep_channel_resources(plchan, txd); if (ret) return NULL; - /* - * NB: the channel lock is held at this point so tx_submit() - * must be called in direct succession. - */ return &txd->tx; } @@ -1531,10 +1460,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, /* Controls applicable to inactive channels */ if (cmd == DMA_SLAVE_CONFIG) { - dma_set_runtime_config(chan, - (struct dma_slave_config *) - arg); - return 0; + return dma_set_runtime_config(chan, + (struct dma_slave_config *)arg); } /* @@ -1558,16 +1485,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, * Mark physical channel as free and free any slave * signal */ - if ((plchan->phychan->signal >= 0) && - pl08x->pd->put_signal) { - pl08x->pd->put_signal(plchan); - plchan->phychan->signal = -1; - } - pl08x_put_phy_channel(pl08x, plchan->phychan); - plchan->phychan = NULL; + release_phy_channel(plchan); } - /* Stop any pending tasklet */ - tasklet_disable(&plchan->tasklet); /* Dequeue jobs and free LLIs */ if (plchan->at) { pl08x_free_txd(pl08x, plchan->at); @@ -1620,78 +1539,71 @@ static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) val = readl(pl08x->base + PL080_CONFIG); val &= ~(PL080_CONFIG_M2_BE | PL080_CONFIG_M1_BE | PL080_CONFIG_ENABLE); - /* We implictly clear bit 1 and that means little-endian mode */ + /* We implicitly clear bit 1 and that means little-endian mode */ val |= PL080_CONFIG_ENABLE; writel(val, pl08x->base + PL080_CONFIG); } +static void pl08x_unmap_buffers(struct pl08x_txd *txd) +{ + struct device *dev = txd->tx.chan->device->dev; + + if (!(txd->tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { + if (txd->tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE) + dma_unmap_single(dev, txd->src_addr, txd->len, + DMA_TO_DEVICE); + else + dma_unmap_page(dev, txd->src_addr, txd->len, + DMA_TO_DEVICE); + } + if (!(txd->tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { + if (txd->tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE) + dma_unmap_single(dev, txd->dst_addr, txd->len, + DMA_FROM_DEVICE); + else + dma_unmap_page(dev, txd->dst_addr, txd->len, + DMA_FROM_DEVICE); + } +} + static void pl08x_tasklet(unsigned long data) { struct pl08x_dma_chan *plchan = (struct pl08x_dma_chan *) data; - struct pl08x_phy_chan *phychan = plchan->phychan; struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_txd *txd; + unsigned long flags; - if (!plchan) - BUG(); + spin_lock_irqsave(&plchan->lock, flags); - spin_lock(&plchan->lock); - - if (plchan->at) { - dma_async_tx_callback callback = - plchan->at->tx.callback; - void *callback_param = - plchan->at->tx.callback_param; + txd = plchan->at; + plchan->at = NULL; + if (txd) { /* * Update last completed */ - plchan->lc = - (plchan->at->tx.cookie); - - /* - * Callback to signal completion - */ - if (callback) - callback(callback_param); - - /* - * Device callbacks should NOT clear - * the current transaction on the channel - * Linus: sometimes they should? - */ - if (!plchan->at) - BUG(); - - /* - * Free the descriptor if it's not for a device - * using a circular buffer - */ - if (!plchan->at->cd->circular_buffer) { - pl08x_free_txd(pl08x, plchan->at); - plchan->at = NULL; - } - /* - * else descriptor for circular - * buffers only freed when - * client has disabled dma - */ + plchan->lc = txd->tx.cookie; } + /* * If a new descriptor is queued, set it up * plchan->at is NULL here */ - if (!list_empty(&plchan->desc_list)) { + if (!list_empty(&plchan->pend_list)) { struct pl08x_txd *next; - next = list_first_entry(&plchan->desc_list, + next = list_first_entry(&plchan->pend_list, struct pl08x_txd, node); list_del(&next->node); - plchan->at = next; - /* Configure the physical channel for the next txd */ - pl08x_config_phychan_for_txd(plchan); - pl08x_set_cregs(pl08x, plchan->phychan); - pl08x_enable_phy_chan(pl08x, plchan->phychan); + + pl08x_start_txd(plchan, next); + } else if (plchan->phychan_hold) { + /* + * This channel is still in use - we have a new txd being + * prepared and will soon be queued. Don't give up the + * physical channel. + */ } else { struct pl08x_dma_chan *waiting = NULL; @@ -1699,12 +1611,7 @@ static void pl08x_tasklet(unsigned long data) * No more jobs, so free up the physical channel * Free any allocated signal on slave transfers too */ - if ((phychan->signal >= 0) && pl08x->pd->put_signal) { - pl08x->pd->put_signal(plchan); - phychan->signal = -1; - } - pl08x_put_phy_channel(pl08x, phychan); - plchan->phychan = NULL; + release_phy_channel(plchan); plchan->state = PL08X_CHAN_IDLE; /* @@ -1724,6 +1631,7 @@ static void pl08x_tasklet(unsigned long data) ret = prep_phy_channel(waiting, waiting->waiting); BUG_ON(ret); + waiting->phychan_hold--; waiting->state = PL08X_CHAN_RUNNING; waiting->waiting = NULL; pl08x_issue_pending(&waiting->chan); @@ -1732,7 +1640,25 @@ static void pl08x_tasklet(unsigned long data) } } - spin_unlock(&plchan->lock); + spin_unlock_irqrestore(&plchan->lock, flags); + + if (txd) { + dma_async_tx_callback callback = txd->tx.callback; + void *callback_param = txd->tx.callback_param; + + /* Don't try to unmap buffers on slave channels */ + if (!plchan->slave) + pl08x_unmap_buffers(txd); + + /* Free the descriptor */ + spin_lock_irqsave(&plchan->lock, flags); + pl08x_free_txd(pl08x, txd); + spin_unlock_irqrestore(&plchan->lock, flags); + + /* Callback to signal completion */ + if (callback) + callback(callback_param); + } } static irqreturn_t pl08x_irq(int irq, void *dev) @@ -1819,16 +1745,23 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, return -ENOMEM; } } + if (chan->cd->circular_buffer) { + dev_err(&pl08x->adev->dev, + "channel %s: circular buffers not supported\n", + chan->name); + kfree(chan); + continue; + } dev_info(&pl08x->adev->dev, "initialize virtual channel \"%s\"\n", chan->name); chan->chan.device = dmadev; - atomic_set(&chan->last_issued, 0); - chan->lc = atomic_read(&chan->last_issued); + chan->chan.cookie = 0; + chan->lc = 0; spin_lock_init(&chan->lock); - INIT_LIST_HEAD(&chan->desc_list); + INIT_LIST_HEAD(&chan->pend_list); tasklet_init(&chan->tasklet, pl08x_tasklet, (unsigned long) chan); @@ -1898,7 +1831,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data) seq_printf(s, "CHANNEL:\tSTATE:\n"); seq_printf(s, "--------\t------\n"); list_for_each_entry(chan, &pl08x->memcpy.channels, chan.device_node) { - seq_printf(s, "%s\t\t\%s\n", chan->name, + seq_printf(s, "%s\t\t%s\n", chan->name, pl08x_state_str(chan->state)); } @@ -1906,7 +1839,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data) seq_printf(s, "CHANNEL:\tSTATE:\n"); seq_printf(s, "--------\t------\n"); list_for_each_entry(chan, &pl08x->slave.channels, chan.device_node) { - seq_printf(s, "%s\t\t\%s\n", chan->name, + seq_printf(s, "%s\t\t%s\n", chan->name, pl08x_state_str(chan->state)); } @@ -1942,7 +1875,7 @@ static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) static int pl08x_probe(struct amba_device *adev, struct amba_id *id) { struct pl08x_driver_data *pl08x; - struct vendor_data *vd = id->data; + const struct vendor_data *vd = id->data; int ret = 0; int i; @@ -1990,6 +1923,14 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id) pl08x->adev = adev; pl08x->vd = vd; + /* By default, AHB1 only. If dualmaster, from platform */ + pl08x->lli_buses = PL08X_AHB1; + pl08x->mem_buses = PL08X_AHB1; + if (pl08x->vd->dualmaster) { + pl08x->lli_buses = pl08x->pd->lli_buses; + pl08x->mem_buses = pl08x->pd->mem_buses; + } + /* A DMA memory pool for LLIs, align on 1-byte boundary */ pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, PL08X_LLI_TSFR_SIZE, PL08X_ALIGN, 0); @@ -2016,7 +1957,7 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id) writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED, - vd->name, pl08x); + DRIVER_NAME, pl08x); if (ret) { dev_err(&adev->dev, "%s failed to request interrupt %d\n", __func__, adev->irq[0]); @@ -2087,8 +2028,9 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id) amba_set_drvdata(adev, pl08x); init_pl08x_debugfs(pl08x); - dev_info(&pl08x->adev->dev, "ARM(R) %s DMA block initialized @%08x\n", - vd->name, adev->res.start); + dev_info(&pl08x->adev->dev, "DMA: PL%03x rev%u at 0x%08llx irq %d\n", + amba_part(adev), amba_rev(adev), + (unsigned long long)adev->res.start, adev->irq[0]); return 0; out_no_slave_reg: @@ -2115,13 +2057,11 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id) /* PL080 has 8 channels and the PL080 have just 2 */ static struct vendor_data vendor_pl080 = { - .name = "PL080", .channels = 8, .dualmaster = true, }; static struct vendor_data vendor_pl081 = { - .name = "PL081", .channels = 2, .dualmaster = false, }; @@ -2160,7 +2100,7 @@ static int __init pl08x_init(void) retval = amba_driver_register(&pl08x_amba_driver); if (retval) printk(KERN_WARNING DRIVER_NAME - "failed to register as an amba device (%d)\n", + "failed to register as an AMBA device (%d)\n", retval); return retval; } diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index fab68a553205..6e1d46a65d0e 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -1,5 +1,6 @@ /* - * Copyright (C) ST-Ericsson SA 2007-2010 + * Copyright (C) Ericsson AB 2007-2008 + * Copyright (C) ST-Ericsson SA 2008-2010 * Author: Per Forlin for ST-Ericsson * Author: Jonas Aaberg for ST-Ericsson * License terms: GNU General Public License (GPL) version 2 @@ -554,8 +555,66 @@ static struct d40_desc *d40_last_queued(struct d40_chan *d40c) return d; } -/* Support functions for logical channels */ +static int d40_psize_2_burst_size(bool is_log, int psize) +{ + if (is_log) { + if (psize == STEDMA40_PSIZE_LOG_1) + return 1; + } else { + if (psize == STEDMA40_PSIZE_PHY_1) + return 1; + } + return 2 << psize; +} + +/* + * The dma only supports transmitting packages up to + * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of + * dma elements required to send the entire sg list + */ +static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2) +{ + int dmalen; + u32 max_w = max(data_width1, data_width2); + u32 min_w = min(data_width1, data_width2); + u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w); + + if (seg_max > STEDMA40_MAX_SEG_SIZE) + seg_max -= (1 << max_w); + + if (!IS_ALIGNED(size, 1 << max_w)) + return -EINVAL; + + if (size <= seg_max) + dmalen = 1; + else { + dmalen = size / seg_max; + if (dmalen * seg_max < size) + dmalen++; + } + return dmalen; +} + +static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len, + u32 data_width1, u32 data_width2) +{ + struct scatterlist *sg; + int i; + int len = 0; + int ret; + + for_each_sg(sgl, sg, sg_len, i) { + ret = d40_size_2_dmalen(sg_dma_len(sg), + data_width1, data_width2); + if (ret < 0) + return ret; + len += ret; + } + return len; +} + +/* Support functions for logical channels */ static int d40_channel_execute_command(struct d40_chan *d40c, enum d40_command command) @@ -1241,6 +1300,21 @@ static int d40_validate_conf(struct d40_chan *d40c, res = -EINVAL; } + if (d40_psize_2_burst_size(is_log, conf->src_info.psize) * + (1 << conf->src_info.data_width) != + d40_psize_2_burst_size(is_log, conf->dst_info.psize) * + (1 << conf->dst_info.data_width)) { + /* + * The DMAC hardware only supports + * src (burst x width) == dst (burst x width) + */ + + dev_err(&d40c->chan.dev->device, + "[%s] src (burst x width) != dst (burst x width)\n", + __func__); + res = -EINVAL; + } + return res; } @@ -1638,13 +1712,21 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, if (d40d == NULL) goto err; - d40d->lli_len = sgl_len; + d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + goto err; + } + d40d->lli_current = 0; d40d->txd.flags = dma_flags; if (d40c->log_num != D40_PHY_CHAN) { - if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; @@ -1654,15 +1736,17 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, sgl_len, d40d->lli_log.src, d40c->log_def.lcsp1, - d40c->dma_cfg.src_info.data_width); + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); (void) d40_log_sg_to_lli(sgl_dst, sgl_len, d40d->lli_log.dst, d40c->log_def.lcsp3, - d40c->dma_cfg.dst_info.data_width); + d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width); } else { - if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; @@ -1675,6 +1759,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, virt_to_phys(d40d->lli_phy.src), d40c->src_def_cfg, d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, d40c->dma_cfg.src_info.psize); if (res < 0) @@ -1687,6 +1772,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, virt_to_phys(d40d->lli_phy.dst), d40c->dst_def_cfg, d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width, d40c->dma_cfg.dst_info.psize); if (res < 0) @@ -1826,7 +1912,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); unsigned long flags; - int err = 0; if (d40c->phy_chan == NULL) { dev_err(&d40c->chan.dev->device, @@ -1844,6 +1929,15 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, } d40d->txd.flags = dma_flags; + d40d->lli_len = d40_size_2_dmalen(size, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + goto err; + } + dma_async_tx_descriptor_init(&d40d->txd, chan); @@ -1851,37 +1945,40 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, if (d40c->log_num != D40_PHY_CHAN) { - if (d40_pool_lli_alloc(d40d, 1, true) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; } - d40d->lli_len = 1; d40d->lli_current = 0; - d40_log_fill_lli(d40d->lli_log.src, - src, - size, - d40c->log_def.lcsp1, - d40c->dma_cfg.src_info.data_width, - true); + if (d40_log_buf_to_lli(d40d->lli_log.src, + src, + size, + d40c->log_def.lcsp1, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, + true) == NULL) + goto err; - d40_log_fill_lli(d40d->lli_log.dst, - dst, - size, - d40c->log_def.lcsp3, - d40c->dma_cfg.dst_info.data_width, - true); + if (d40_log_buf_to_lli(d40d->lli_log.dst, + dst, + size, + d40c->log_def.lcsp3, + d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width, + true) == NULL) + goto err; } else { - if (d40_pool_lli_alloc(d40d, 1, false) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; } - err = d40_phy_fill_lli(d40d->lli_phy.src, + if (d40_phy_buf_to_lli(d40d->lli_phy.src, src, size, d40c->dma_cfg.src_info.psize, @@ -1889,11 +1986,11 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, d40c->src_def_cfg, true, d40c->dma_cfg.src_info.data_width, - false); - if (err) - goto err_fill_lli; + d40c->dma_cfg.dst_info.data_width, + false) == NULL) + goto err; - err = d40_phy_fill_lli(d40d->lli_phy.dst, + if (d40_phy_buf_to_lli(d40d->lli_phy.dst, dst, size, d40c->dma_cfg.dst_info.psize, @@ -1901,10 +1998,9 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, d40c->dst_def_cfg, true, d40c->dma_cfg.dst_info.data_width, - false); - - if (err) - goto err_fill_lli; + d40c->dma_cfg.src_info.data_width, + false) == NULL) + goto err; (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src, d40d->lli_pool.size, DMA_TO_DEVICE); @@ -1913,9 +2009,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, spin_unlock_irqrestore(&d40c->lock, flags); return &d40d->txd; -err_fill_lli: - dev_err(&d40c->chan.dev->device, - "[%s] Failed filling in PHY LLI\n", __func__); err: if (d40d) d40_desc_free(d40c, d40d); @@ -1945,13 +2038,21 @@ static int d40_prep_slave_sg_log(struct d40_desc *d40d, dma_addr_t dev_addr = 0; int total_size; - if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) { + d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + return -EINVAL; + } + + if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); return -ENOMEM; } - d40d->lli_len = sg_len; d40d->lli_current = 0; if (direction == DMA_FROM_DEVICE) @@ -1993,13 +2094,21 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d, dma_addr_t dst_dev_addr; int res; - if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { + d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + return -EINVAL; + } + + if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); return -ENOMEM; } - d40d->lli_len = sgl_len; d40d->lli_current = 0; if (direction == DMA_FROM_DEVICE) { @@ -2024,6 +2133,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d, virt_to_phys(d40d->lli_phy.src), d40c->src_def_cfg, d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, d40c->dma_cfg.src_info.psize); if (res < 0) return res; @@ -2035,6 +2145,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d, virt_to_phys(d40d->lli_phy.dst), d40c->dst_def_cfg, d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width, d40c->dma_cfg.dst_info.psize); if (res < 0) return res; @@ -2244,6 +2355,8 @@ static void d40_set_runtime_config(struct dma_chan *chan, psize = STEDMA40_PSIZE_PHY_8; else if (config_maxburst >= 4) psize = STEDMA40_PSIZE_PHY_4; + else if (config_maxburst >= 2) + psize = STEDMA40_PSIZE_PHY_2; else psize = STEDMA40_PSIZE_PHY_1; } diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c index 8557cb88b255..0b096a38322d 100644 --- a/drivers/dma/ste_dma40_ll.c +++ b/drivers/dma/ste_dma40_ll.c @@ -1,6 +1,6 @@ /* * Copyright (C) ST-Ericsson SA 2007-2010 - * Author: Per Friden for ST-Ericsson + * Author: Per Forlin for ST-Ericsson * Author: Jonas Aaberg for ST-Ericsson * License terms: GNU General Public License (GPL) version 2 */ @@ -122,15 +122,15 @@ void d40_phy_cfg(struct stedma40_chan_cfg *cfg, *dst_cfg = dst; } -int d40_phy_fill_lli(struct d40_phy_lli *lli, - dma_addr_t data, - u32 data_size, - int psize, - dma_addr_t next_lli, - u32 reg_cfg, - bool term_int, - u32 data_width, - bool is_device) +static int d40_phy_fill_lli(struct d40_phy_lli *lli, + dma_addr_t data, + u32 data_size, + int psize, + dma_addr_t next_lli, + u32 reg_cfg, + bool term_int, + u32 data_width, + bool is_device) { int num_elems; @@ -139,13 +139,6 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli, else num_elems = 2 << psize; - /* - * Size is 16bit. data_width is 8, 16, 32 or 64 bit - * Block large than 64 KiB must be split. - */ - if (data_size > (0xffff << data_width)) - return -EINVAL; - /* Must be aligned */ if (!IS_ALIGNED(data, 0x1 << data_width)) return -EINVAL; @@ -187,55 +180,118 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli, return 0; } +static int d40_seg_size(int size, int data_width1, int data_width2) +{ + u32 max_w = max(data_width1, data_width2); + u32 min_w = min(data_width1, data_width2); + u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w); + + if (seg_max > STEDMA40_MAX_SEG_SIZE) + seg_max -= (1 << max_w); + + if (size <= seg_max) + return size; + + if (size <= 2 * seg_max) + return ALIGN(size / 2, 1 << max_w); + + return seg_max; +} + +struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli, + dma_addr_t addr, + u32 size, + int psize, + dma_addr_t lli_phys, + u32 reg_cfg, + bool term_int, + u32 data_width1, + u32 data_width2, + bool is_device) +{ + int err; + dma_addr_t next = lli_phys; + int size_rest = size; + int size_seg = 0; + + do { + size_seg = d40_seg_size(size_rest, data_width1, data_width2); + size_rest -= size_seg; + + if (term_int && size_rest == 0) + next = 0; + else + next = ALIGN(next + sizeof(struct d40_phy_lli), + D40_LLI_ALIGN); + + err = d40_phy_fill_lli(lli, + addr, + size_seg, + psize, + next, + reg_cfg, + !next, + data_width1, + is_device); + + if (err) + goto err; + + lli++; + if (!is_device) + addr += size_seg; + } while (size_rest); + + return lli; + + err: + return NULL; +} + int d40_phy_sg_to_lli(struct scatterlist *sg, int sg_len, dma_addr_t target, - struct d40_phy_lli *lli, + struct d40_phy_lli *lli_sg, dma_addr_t lli_phys, u32 reg_cfg, - u32 data_width, + u32 data_width1, + u32 data_width2, int psize) { int total_size = 0; int i; struct scatterlist *current_sg = sg; - dma_addr_t next_lli_phys; dma_addr_t dst; - int err = 0; + struct d40_phy_lli *lli = lli_sg; + dma_addr_t l_phys = lli_phys; for_each_sg(sg, current_sg, sg_len, i) { total_size += sg_dma_len(current_sg); - /* If this scatter list entry is the last one, no next link */ - if (sg_len - 1 == i) - next_lli_phys = 0; - else - next_lli_phys = ALIGN(lli_phys + (i + 1) * - sizeof(struct d40_phy_lli), - D40_LLI_ALIGN); - if (target) dst = target; else dst = sg_phys(current_sg); - err = d40_phy_fill_lli(&lli[i], - dst, - sg_dma_len(current_sg), - psize, - next_lli_phys, - reg_cfg, - !next_lli_phys, - data_width, - target == dst); - if (err) - goto err; + l_phys = ALIGN(lli_phys + (lli - lli_sg) * + sizeof(struct d40_phy_lli), D40_LLI_ALIGN); + + lli = d40_phy_buf_to_lli(lli, + dst, + sg_dma_len(current_sg), + psize, + l_phys, + reg_cfg, + sg_len - 1 == i, + data_width1, + data_width2, + target == dst); + if (lli == NULL) + return -EINVAL; } return total_size; -err: - return err; } @@ -315,17 +371,20 @@ void d40_log_lli_lcla_write(struct d40_log_lli *lcla, writel(lli_dst->lcsp13, &lcla[1].lcsp13); } -void d40_log_fill_lli(struct d40_log_lli *lli, - dma_addr_t data, u32 data_size, - u32 reg_cfg, - u32 data_width, - bool addr_inc) +static void d40_log_fill_lli(struct d40_log_lli *lli, + dma_addr_t data, u32 data_size, + u32 reg_cfg, + u32 data_width, + bool addr_inc) { lli->lcsp13 = reg_cfg; /* The number of elements to transfer */ lli->lcsp02 = ((data_size >> data_width) << D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK; + + BUG_ON((data_size >> data_width) > STEDMA40_MAX_SEG_SIZE); + /* 16 LSBs address of the current element */ lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK; /* 16 MSBs address of the current element */ @@ -348,55 +407,94 @@ int d40_log_sg_to_dev(struct scatterlist *sg, int total_size = 0; struct scatterlist *current_sg = sg; int i; + struct d40_log_lli *lli_src = lli->src; + struct d40_log_lli *lli_dst = lli->dst; for_each_sg(sg, current_sg, sg_len, i) { total_size += sg_dma_len(current_sg); if (direction == DMA_TO_DEVICE) { - d40_log_fill_lli(&lli->src[i], - sg_phys(current_sg), - sg_dma_len(current_sg), - lcsp->lcsp1, src_data_width, - true); - d40_log_fill_lli(&lli->dst[i], - dev_addr, - sg_dma_len(current_sg), - lcsp->lcsp3, dst_data_width, - false); + lli_src = + d40_log_buf_to_lli(lli_src, + sg_phys(current_sg), + sg_dma_len(current_sg), + lcsp->lcsp1, src_data_width, + dst_data_width, + true); + lli_dst = + d40_log_buf_to_lli(lli_dst, + dev_addr, + sg_dma_len(current_sg), + lcsp->lcsp3, dst_data_width, + src_data_width, + false); } else { - d40_log_fill_lli(&lli->dst[i], - sg_phys(current_sg), - sg_dma_len(current_sg), - lcsp->lcsp3, dst_data_width, - true); - d40_log_fill_lli(&lli->src[i], - dev_addr, - sg_dma_len(current_sg), - lcsp->lcsp1, src_data_width, - false); + lli_dst = + d40_log_buf_to_lli(lli_dst, + sg_phys(current_sg), + sg_dma_len(current_sg), + lcsp->lcsp3, dst_data_width, + src_data_width, + true); + lli_src = + d40_log_buf_to_lli(lli_src, + dev_addr, + sg_dma_len(current_sg), + lcsp->lcsp1, src_data_width, + dst_data_width, + false); } } return total_size; } +struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg, + dma_addr_t addr, + int size, + u32 lcsp13, /* src or dst*/ + u32 data_width1, + u32 data_width2, + bool addr_inc) +{ + struct d40_log_lli *lli = lli_sg; + int size_rest = size; + int size_seg = 0; + + do { + size_seg = d40_seg_size(size_rest, data_width1, data_width2); + size_rest -= size_seg; + + d40_log_fill_lli(lli, + addr, + size_seg, + lcsp13, data_width1, + addr_inc); + if (addr_inc) + addr += size_seg; + lli++; + } while (size_rest); + + return lli; +} + int d40_log_sg_to_lli(struct scatterlist *sg, int sg_len, struct d40_log_lli *lli_sg, u32 lcsp13, /* src or dst*/ - u32 data_width) + u32 data_width1, u32 data_width2) { int total_size = 0; struct scatterlist *current_sg = sg; int i; + struct d40_log_lli *lli = lli_sg; for_each_sg(sg, current_sg, sg_len, i) { total_size += sg_dma_len(current_sg); - - d40_log_fill_lli(&lli_sg[i], - sg_phys(current_sg), - sg_dma_len(current_sg), - lcsp13, data_width, - true); + lli = d40_log_buf_to_lli(lli, + sg_phys(current_sg), + sg_dma_len(current_sg), + lcsp13, + data_width1, data_width2, true); } return total_size; } diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h index 9e419b907544..9cc43495bea2 100644 --- a/drivers/dma/ste_dma40_ll.h +++ b/drivers/dma/ste_dma40_ll.h @@ -292,18 +292,20 @@ int d40_phy_sg_to_lli(struct scatterlist *sg, struct d40_phy_lli *lli, dma_addr_t lli_phys, u32 reg_cfg, - u32 data_width, + u32 data_width1, + u32 data_width2, int psize); -int d40_phy_fill_lli(struct d40_phy_lli *lli, - dma_addr_t data, - u32 data_size, - int psize, - dma_addr_t next_lli, - u32 reg_cfg, - bool term_int, - u32 data_width, - bool is_device); +struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli, + dma_addr_t data, + u32 data_size, + int psize, + dma_addr_t next_lli, + u32 reg_cfg, + bool term_int, + u32 data_width1, + u32 data_width2, + bool is_device); void d40_phy_lli_write(void __iomem *virtbase, u32 phy_chan_num, @@ -312,12 +314,12 @@ void d40_phy_lli_write(void __iomem *virtbase, /* Logical channels */ -void d40_log_fill_lli(struct d40_log_lli *lli, - dma_addr_t data, - u32 data_size, - u32 reg_cfg, - u32 data_width, - bool addr_inc); +struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg, + dma_addr_t addr, + int size, + u32 lcsp13, /* src or dst*/ + u32 data_width1, u32 data_width2, + bool addr_inc); int d40_log_sg_to_dev(struct scatterlist *sg, int sg_len, @@ -332,7 +334,7 @@ int d40_log_sg_to_lli(struct scatterlist *sg, int sg_len, struct d40_log_lli *lli_sg, u32 lcsp13, /* src or dst*/ - u32 data_width); + u32 data_width1, u32 data_width2); void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, struct d40_log_lli *lli_dst, diff --git a/include/linux/amba/pl08x.h b/include/linux/amba/pl08x.h index 521a0f8974ac..933b4ed12be5 100644 --- a/include/linux/amba/pl08x.h +++ b/include/linux/amba/pl08x.h @@ -22,6 +22,15 @@ #include #include +struct pl08x_lli; +struct pl08x_driver_data; + +/* Bitmasks for selecting AHB ports for DMA transfers */ +enum { + PL08X_AHB1 = (1 << 0), + PL08X_AHB2 = (1 << 1) +}; + /** * struct pl08x_channel_data - data structure to pass info between * platform and PL08x driver regarding channel configuration @@ -48,6 +57,8 @@ * round round round) * @single: the device connected to this channel will request single * DMA transfers, not bursts. (Bursts are default.) + * @periph_buses: the device connected to this channel is accessible via + * these buses (use PL08X_AHB1 | PL08X_AHB2). */ struct pl08x_channel_data { char *bus_id; @@ -55,10 +66,10 @@ struct pl08x_channel_data { int max_signal; u32 muxval; u32 cctl; - u32 ccfg; dma_addr_t addr; bool circular_buffer; bool single; + u8 periph_buses; }; /** @@ -74,7 +85,7 @@ struct pl08x_bus_data { dma_addr_t addr; u8 maxwidth; u8 buswidth; - u32 fill_bytes; + size_t fill_bytes; }; /** @@ -92,11 +103,6 @@ struct pl08x_phy_chan { spinlock_t lock; int signal; struct pl08x_dma_chan *serving; - u32 csrc; - u32 cdst; - u32 clli; - u32 cctl; - u32 ccfg; }; /** @@ -108,21 +114,19 @@ struct pl08x_txd { struct dma_async_tx_descriptor tx; struct list_head node; enum dma_data_direction direction; - struct pl08x_bus_data srcbus; - struct pl08x_bus_data dstbus; - int len; + dma_addr_t src_addr; + dma_addr_t dst_addr; + size_t len; dma_addr_t llis_bus; void *llis_va; - struct pl08x_channel_data *cd; bool active; + /* Default cctl value for LLIs */ + u32 cctl; /* * Settings to be put into the physical channel when we - * trigger this txd + * trigger this txd. Other registers are in llis_va[0]. */ - u32 csrc; - u32 cdst; - u32 clli; - u32 cctl; + u32 ccfg; }; /** @@ -147,6 +151,8 @@ enum pl08x_dma_chan_state { * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel * @chan: wrappped abstract channel * @phychan: the physical channel utilized by this channel, if there is one + * @phychan_hold: if non-zero, hold on to the physical channel even if we + * have no pending entries * @tasklet: tasklet scheduled by the IRQ to handle actual work etc * @name: name of channel * @cd: channel platform data @@ -154,11 +160,8 @@ enum pl08x_dma_chan_state { * @runtime_direction: current direction of this channel according to * runtime config * @lc: last completed transaction on this channel - * @desc_list: queued transactions pending on this channel + * @pend_list: queued transactions pending on this channel * @at: active transaction on this channel - * @lockflags: sometimes we let a lock last between two function calls, - * especially prep/submit, and then we need to store the IRQ flags - * in the channel state, here * @lock: a lock for this channel data * @host: a pointer to the host (internal use) * @state: whether the channel is idle, paused, running etc @@ -169,18 +172,17 @@ enum pl08x_dma_chan_state { struct pl08x_dma_chan { struct dma_chan chan; struct pl08x_phy_chan *phychan; + int phychan_hold; struct tasklet_struct tasklet; char *name; struct pl08x_channel_data *cd; dma_addr_t runtime_addr; enum dma_data_direction runtime_direction; - atomic_t last_issued; dma_cookie_t lc; - struct list_head desc_list; + struct list_head pend_list; struct pl08x_txd *at; - unsigned long lockflags; spinlock_t lock; - void *host; + struct pl08x_driver_data *host; enum pl08x_dma_chan_state state; bool slave; struct pl08x_txd *waiting; @@ -199,8 +201,8 @@ struct pl08x_dma_chan { * less than zero, else it returns the allocated signal number * @put_signal: indicate to the platform that this physical signal is not * running any DMA transfer and multiplexing can be recycled - * @bus_bit_lli: Bit[0] of the address indicated which AHB bus master the - * LLI addresses are on 0/1 Master 1/2. + * @lli_buses: buses which LLIs can be fetched from: PL08X_AHB1 | PL08X_AHB2 + * @mem_buses: buses which memory can be accessed from: PL08X_AHB1 | PL08X_AHB2 */ struct pl08x_platform_data { struct pl08x_channel_data *slave_channels; @@ -208,6 +210,8 @@ struct pl08x_platform_data { struct pl08x_channel_data memcpy_channel; int (*get_signal)(struct pl08x_dma_chan *); void (*put_signal)(struct pl08x_dma_chan *); + u8 lli_buses; + u8 mem_buses; }; #ifdef CONFIG_AMBA_PL08X