2011-03-13 16:54:26 +08:00
|
|
|
/*
|
|
|
|
* CAAM/SEC 4.x transport/backend driver
|
|
|
|
* JobR backend functionality
|
|
|
|
*
|
2012-06-23 08:48:54 +08:00
|
|
|
* Copyright 2008-2012 Freescale Semiconductor, Inc.
|
2011-03-13 16:54:26 +08:00
|
|
|
*/
|
|
|
|
|
2013-09-18 03:28:33 +08:00
|
|
|
#include <linux/of_irq.h>
|
2013-11-18 12:20:01 +08:00
|
|
|
#include <linux/of_address.h>
|
2013-09-18 03:28:33 +08:00
|
|
|
|
2011-03-13 16:54:26 +08:00
|
|
|
#include "compat.h"
|
|
|
|
#include "regs.h"
|
|
|
|
#include "jr.h"
|
|
|
|
#include "desc.h"
|
|
|
|
#include "intern.h"
|
|
|
|
|
2013-10-25 14:31:01 +08:00
|
|
|
struct jr_driver_data {
|
|
|
|
/* List of Physical JobR's with the Driver */
|
|
|
|
struct list_head jr_list;
|
|
|
|
spinlock_t jr_alloc_lock; /* jr_list lock */
|
|
|
|
} ____cacheline_aligned;
|
|
|
|
|
|
|
|
static struct jr_driver_data driver_data;
|
|
|
|
|
|
|
|
static int caam_reset_hw_jr(struct device *dev)
|
|
|
|
{
|
|
|
|
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
|
|
|
|
unsigned int timeout = 100000;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* mask interrupts since we are going to poll
|
|
|
|
* for reset completion status
|
|
|
|
*/
|
|
|
|
setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
|
|
|
|
|
|
|
|
/* initiate flush (required prior to reset) */
|
|
|
|
wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
|
|
|
|
while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
|
|
|
|
JRINT_ERR_HALT_INPROGRESS) && --timeout)
|
|
|
|
cpu_relax();
|
|
|
|
|
|
|
|
if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
|
|
|
|
JRINT_ERR_HALT_COMPLETE || timeout == 0) {
|
|
|
|
dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* initiate reset */
|
|
|
|
timeout = 100000;
|
|
|
|
wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
|
|
|
|
while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
|
|
|
|
cpu_relax();
|
|
|
|
|
|
|
|
if (timeout == 0) {
|
|
|
|
dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* unmask interrupts */
|
|
|
|
clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Shutdown JobR independent of platform property code
|
|
|
|
*/
|
|
|
|
int caam_jr_shutdown(struct device *dev)
|
|
|
|
{
|
|
|
|
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
|
|
|
|
dma_addr_t inpbusaddr, outbusaddr;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = caam_reset_hw_jr(dev);
|
|
|
|
|
|
|
|
tasklet_kill(&jrp->irqtask);
|
|
|
|
|
|
|
|
/* Release interrupt */
|
|
|
|
free_irq(jrp->irq, dev);
|
|
|
|
|
|
|
|
/* Free rings */
|
|
|
|
inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
|
|
|
|
outbusaddr = rd_reg64(&jrp->rregs->outring_base);
|
|
|
|
dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
|
|
|
|
jrp->inpring, inpbusaddr);
|
|
|
|
dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
|
|
|
|
jrp->outring, outbusaddr);
|
|
|
|
kfree(jrp->entinfo);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int caam_jr_remove(struct platform_device *pdev)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
struct device *jrdev;
|
|
|
|
struct caam_drv_private_jr *jrpriv;
|
|
|
|
|
|
|
|
jrdev = &pdev->dev;
|
|
|
|
jrpriv = dev_get_drvdata(jrdev);
|
|
|
|
|
|
|
|
/*
|
2013-10-25 14:31:02 +08:00
|
|
|
* Return EBUSY if job ring already allocated.
|
2013-10-25 14:31:01 +08:00
|
|
|
*/
|
2013-10-25 14:31:02 +08:00
|
|
|
if (atomic_read(&jrpriv->tfm_count)) {
|
2013-10-25 14:31:01 +08:00
|
|
|
dev_err(jrdev, "Device is busy\n");
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Remove the node from Physical JobR list maintained by driver */
|
|
|
|
spin_lock(&driver_data.jr_alloc_lock);
|
|
|
|
list_del(&jrpriv->list_node);
|
|
|
|
spin_unlock(&driver_data.jr_alloc_lock);
|
|
|
|
|
|
|
|
/* Release ring */
|
|
|
|
ret = caam_jr_shutdown(jrdev);
|
|
|
|
if (ret)
|
|
|
|
dev_err(jrdev, "Failed to shut down job ring\n");
|
|
|
|
irq_dispose_mapping(jrpriv->irq);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2011-03-13 16:54:26 +08:00
|
|
|
/* Main per-ring interrupt handler */
|
|
|
|
static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
|
|
|
|
{
|
|
|
|
struct device *dev = st_dev;
|
|
|
|
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
|
|
|
|
u32 irqstate;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check the output ring for ready responses, kick
|
|
|
|
* tasklet if jobs done.
|
|
|
|
*/
|
|
|
|
irqstate = rd_reg32(&jrp->rregs->jrintstatus);
|
|
|
|
if (!irqstate)
|
|
|
|
return IRQ_NONE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If JobR error, we got more development work to do
|
|
|
|
* Flag a bug now, but we really need to shut down and
|
|
|
|
* restart the queue (and fix code).
|
|
|
|
*/
|
|
|
|
if (irqstate & JRINT_JR_ERROR) {
|
|
|
|
dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
|
|
|
|
/* mask valid interrupts */
|
|
|
|
setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
|
|
|
|
|
|
|
|
/* Have valid interrupt at this point, just ACK and trigger */
|
|
|
|
wr_reg32(&jrp->rregs->jrintstatus, irqstate);
|
|
|
|
|
|
|
|
preempt_disable();
|
2012-06-23 08:48:57 +08:00
|
|
|
tasklet_schedule(&jrp->irqtask);
|
2011-03-13 16:54:26 +08:00
|
|
|
preempt_enable();
|
|
|
|
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Deferred service handler, run as interrupt-fired tasklet */
|
|
|
|
static void caam_jr_dequeue(unsigned long devarg)
|
|
|
|
{
|
|
|
|
int hw_idx, sw_idx, i, head, tail;
|
|
|
|
struct device *dev = (struct device *)devarg;
|
|
|
|
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
|
|
|
|
void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
|
|
|
|
u32 *userdesc, userstatus;
|
|
|
|
void *userarg;
|
|
|
|
|
2012-06-23 08:48:55 +08:00
|
|
|
while (rd_reg32(&jrp->rregs->outring_used)) {
|
2011-03-13 16:54:26 +08:00
|
|
|
|
2012-06-23 08:48:55 +08:00
|
|
|
head = ACCESS_ONCE(jrp->head);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
crypto: caam - fix possible deadlock condition
commit "crypto: caam - use non-irq versions of spinlocks for job rings"
made two bad assumptions:
(a) The caam_jr_enqueue lock isn't used in softirq context.
Not true: jr_enqueue can be interrupted by an incoming net
interrupt and the received packet may be sent for encryption,
via caam_jr_enqueue in softirq context, thereby inducing a
deadlock.
This is evidenced when running netperf over an IPSec tunnel
between two P4080's, with spinlock debugging turned on:
[ 892.092569] BUG: spinlock lockup on CPU#7, netperf/10634, e8bf5f70
[ 892.098747] Call Trace:
[ 892.101197] [eff9fc10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.107563] [eff9fc50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.113399] [eff9fc80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.118889] [eff9fc90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.124550] [eff9fcd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129625] BUG: spinlock lockup on CPU#5, swapper/5/0, e8bf5f70
[ 892.129629] Call Trace:
[ 892.129637] [effa7c10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.129645] [effa7c50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.129652] [effa7c80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.129660] [effa7c90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.129666] [effa7cd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129674] [effa7d00] [c0509724] esp_input+0x178/0x334
[ 892.129681] [effa7d50] [c0519778] xfrm_input+0x77c/0x818
[ 892.129688] [effa7da0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.129697] [effa7db0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.129703] [effa7de0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.129709] [effa7e10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.129715] [effa7e80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.129726] [effa7eb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.129732] [effa7ef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.129742] [effa7f10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.129748] [effa7f40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.129754] [effa7f60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.129763] [effa7fa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.129769] [effa7ff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.129775] [ebacfe70] [c0004868] do_softirq+0xd8/0x104
[ 892.129780] [ebacfe90] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.129786] [ebacfea0] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.129792] [ebacfed0] [c000fad8] ret_from_except+0x0/0x18
[ 892.129798] [ebacff90] [c0009010] cpu_idle+0x94/0xf0
[ 892.129804] [ebacffb0] [c059ff88] start_secondary+0x42c/0x430
[ 892.129809] [ebacfff0] [c0001e28] __secondary_start+0x30/0x84
[ 892.281474]
[ 892.282959] [eff9fd00] [c0509724] esp_input+0x178/0x334
[ 892.288186] [eff9fd50] [c0519778] xfrm_input+0x77c/0x818
[ 892.293499] [eff9fda0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.299074] [eff9fdb0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.304907] [eff9fde0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.309872] [eff9fe10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.315966] [eff9fe80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.321803] [eff9feb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.326855] [eff9fef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.333212] [eff9ff10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.338872] [eff9ff40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.344271] [eff9ff60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.349846] [eff9ffa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.355338] [eff9fff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.360910] [e7169950] [c0004868] do_softirq+0xd8/0x104
[ 892.366135] [e7169970] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.371101] [e7169980] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.375979] [e71699b0] [c000fad8] ret_from_except+0x0/0x18
[ 892.381466] [e7169a70] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.387127] [e7169ab0] [c044ad4c] aead_givencrypt+0x6ac/0xa70
[ 892.392873] [e7169b20] [c050a0b8] esp_output+0x2b4/0x570
[ 892.398186] [e7169b80] [c0519b9c] xfrm_output_resume+0x248/0x7c0
[ 892.404194] [e7169bb0] [c050e89c] xfrm4_output_finish+0x18/0x28
[ 892.410113] [e7169bc0] [c050e8f4] xfrm4_output+0x48/0x98
[ 892.415427] [e7169bd0] [c04beac0] ip_local_out+0x48/0x98
[ 892.420740] [e7169be0] [c04bec7c] ip_queue_xmit+0x16c/0x490
[ 892.426314] [e7169c10] [c04d6128] tcp_transmit_skb+0x35c/0x9a4
[ 892.432147] [e7169c70] [c04d6f98] tcp_write_xmit+0x200/0xa04
[ 892.437808] [e7169cc0] [c04c8ccc] tcp_sendmsg+0x994/0xcec
[ 892.443213] [e7169d40] [c04eebfc] inet_sendmsg+0xd0/0x164
[ 892.448617] [e7169d70] [c04792f8] sock_sendmsg+0x8c/0xbc
[ 892.453931] [e7169e40] [c047aecc] sys_sendto+0xc0/0xfc
[ 892.459069] [e7169f10] [c047b934] sys_socketcall+0x110/0x25c
[ 892.464729] [e7169f40] [c000f480] ret_from_syscall+0x0/0x3c
(b) since the caam_jr_dequeue lock is only used in bh context,
then semantically it should use _bh spin_lock types. spin_lock_bh
semantics are to disable back-halves, and used when a lock is shared
between softirq (bh) context and process and/or h/w IRQ context.
Since the lock is only used within softirq context, and this tasklet
is atomic, there is no need to do the additional work to disable
back halves.
This patch adds back-half disabling protection to caam_jr_enqueue
spin_locks to fix (a), and drops it from caam_jr_dequeue to fix (b).
Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2012-07-14 07:04:23 +08:00
|
|
|
spin_lock(&jrp->outlock);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
2012-06-23 08:48:55 +08:00
|
|
|
sw_idx = tail = jrp->tail;
|
2011-03-13 16:54:26 +08:00
|
|
|
hw_idx = jrp->out_ring_read_index;
|
2012-06-23 08:48:55 +08:00
|
|
|
|
2011-03-13 16:54:26 +08:00
|
|
|
for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
|
|
|
|
sw_idx = (tail + i) & (JOBR_DEPTH - 1);
|
|
|
|
|
|
|
|
if (jrp->outring[hw_idx].desc ==
|
|
|
|
jrp->entinfo[sw_idx].desc_addr_dma)
|
|
|
|
break; /* found */
|
|
|
|
}
|
|
|
|
/* we should never fail to find a matching descriptor */
|
|
|
|
BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
|
|
|
|
|
|
|
|
/* Unmap just-run descriptor so we can post-process */
|
|
|
|
dma_unmap_single(dev, jrp->outring[hw_idx].desc,
|
|
|
|
jrp->entinfo[sw_idx].desc_size,
|
|
|
|
DMA_TO_DEVICE);
|
|
|
|
|
|
|
|
/* mark completed, avoid matching on a recycled desc addr */
|
|
|
|
jrp->entinfo[sw_idx].desc_addr_dma = 0;
|
|
|
|
|
|
|
|
/* Stash callback params for use outside of lock */
|
|
|
|
usercall = jrp->entinfo[sw_idx].callbk;
|
|
|
|
userarg = jrp->entinfo[sw_idx].cbkarg;
|
|
|
|
userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
|
|
|
|
userstatus = jrp->outring[hw_idx].jrstatus;
|
|
|
|
|
2015-08-06 02:28:35 +08:00
|
|
|
/*
|
|
|
|
* Make sure all information from the job has been obtained
|
|
|
|
* before telling CAAM that the job has been removed from the
|
|
|
|
* output ring.
|
|
|
|
*/
|
|
|
|
mb();
|
|
|
|
|
2012-06-23 08:48:56 +08:00
|
|
|
/* set done */
|
|
|
|
wr_reg32(&jrp->rregs->outring_rmvd, 1);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
|
|
|
jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
|
|
|
|
(JOBR_DEPTH - 1);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* if this job completed out-of-order, do not increment
|
|
|
|
* the tail. Otherwise, increment tail by 1 plus the
|
|
|
|
* number of subsequent jobs already completed out-of-order
|
|
|
|
*/
|
|
|
|
if (sw_idx == tail) {
|
|
|
|
do {
|
|
|
|
tail = (tail + 1) & (JOBR_DEPTH - 1);
|
|
|
|
} while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
|
|
|
|
jrp->entinfo[tail].desc_addr_dma == 0);
|
|
|
|
|
|
|
|
jrp->tail = tail;
|
|
|
|
}
|
|
|
|
|
crypto: caam - fix possible deadlock condition
commit "crypto: caam - use non-irq versions of spinlocks for job rings"
made two bad assumptions:
(a) The caam_jr_enqueue lock isn't used in softirq context.
Not true: jr_enqueue can be interrupted by an incoming net
interrupt and the received packet may be sent for encryption,
via caam_jr_enqueue in softirq context, thereby inducing a
deadlock.
This is evidenced when running netperf over an IPSec tunnel
between two P4080's, with spinlock debugging turned on:
[ 892.092569] BUG: spinlock lockup on CPU#7, netperf/10634, e8bf5f70
[ 892.098747] Call Trace:
[ 892.101197] [eff9fc10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.107563] [eff9fc50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.113399] [eff9fc80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.118889] [eff9fc90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.124550] [eff9fcd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129625] BUG: spinlock lockup on CPU#5, swapper/5/0, e8bf5f70
[ 892.129629] Call Trace:
[ 892.129637] [effa7c10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.129645] [effa7c50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.129652] [effa7c80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.129660] [effa7c90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.129666] [effa7cd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129674] [effa7d00] [c0509724] esp_input+0x178/0x334
[ 892.129681] [effa7d50] [c0519778] xfrm_input+0x77c/0x818
[ 892.129688] [effa7da0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.129697] [effa7db0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.129703] [effa7de0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.129709] [effa7e10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.129715] [effa7e80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.129726] [effa7eb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.129732] [effa7ef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.129742] [effa7f10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.129748] [effa7f40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.129754] [effa7f60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.129763] [effa7fa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.129769] [effa7ff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.129775] [ebacfe70] [c0004868] do_softirq+0xd8/0x104
[ 892.129780] [ebacfe90] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.129786] [ebacfea0] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.129792] [ebacfed0] [c000fad8] ret_from_except+0x0/0x18
[ 892.129798] [ebacff90] [c0009010] cpu_idle+0x94/0xf0
[ 892.129804] [ebacffb0] [c059ff88] start_secondary+0x42c/0x430
[ 892.129809] [ebacfff0] [c0001e28] __secondary_start+0x30/0x84
[ 892.281474]
[ 892.282959] [eff9fd00] [c0509724] esp_input+0x178/0x334
[ 892.288186] [eff9fd50] [c0519778] xfrm_input+0x77c/0x818
[ 892.293499] [eff9fda0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.299074] [eff9fdb0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.304907] [eff9fde0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.309872] [eff9fe10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.315966] [eff9fe80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.321803] [eff9feb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.326855] [eff9fef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.333212] [eff9ff10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.338872] [eff9ff40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.344271] [eff9ff60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.349846] [eff9ffa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.355338] [eff9fff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.360910] [e7169950] [c0004868] do_softirq+0xd8/0x104
[ 892.366135] [e7169970] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.371101] [e7169980] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.375979] [e71699b0] [c000fad8] ret_from_except+0x0/0x18
[ 892.381466] [e7169a70] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.387127] [e7169ab0] [c044ad4c] aead_givencrypt+0x6ac/0xa70
[ 892.392873] [e7169b20] [c050a0b8] esp_output+0x2b4/0x570
[ 892.398186] [e7169b80] [c0519b9c] xfrm_output_resume+0x248/0x7c0
[ 892.404194] [e7169bb0] [c050e89c] xfrm4_output_finish+0x18/0x28
[ 892.410113] [e7169bc0] [c050e8f4] xfrm4_output+0x48/0x98
[ 892.415427] [e7169bd0] [c04beac0] ip_local_out+0x48/0x98
[ 892.420740] [e7169be0] [c04bec7c] ip_queue_xmit+0x16c/0x490
[ 892.426314] [e7169c10] [c04d6128] tcp_transmit_skb+0x35c/0x9a4
[ 892.432147] [e7169c70] [c04d6f98] tcp_write_xmit+0x200/0xa04
[ 892.437808] [e7169cc0] [c04c8ccc] tcp_sendmsg+0x994/0xcec
[ 892.443213] [e7169d40] [c04eebfc] inet_sendmsg+0xd0/0x164
[ 892.448617] [e7169d70] [c04792f8] sock_sendmsg+0x8c/0xbc
[ 892.453931] [e7169e40] [c047aecc] sys_sendto+0xc0/0xfc
[ 892.459069] [e7169f10] [c047b934] sys_socketcall+0x110/0x25c
[ 892.464729] [e7169f40] [c000f480] ret_from_syscall+0x0/0x3c
(b) since the caam_jr_dequeue lock is only used in bh context,
then semantically it should use _bh spin_lock types. spin_lock_bh
semantics are to disable back-halves, and used when a lock is shared
between softirq (bh) context and process and/or h/w IRQ context.
Since the lock is only used within softirq context, and this tasklet
is atomic, there is no need to do the additional work to disable
back halves.
This patch adds back-half disabling protection to caam_jr_enqueue
spin_locks to fix (a), and drops it from caam_jr_dequeue to fix (b).
Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2012-07-14 07:04:23 +08:00
|
|
|
spin_unlock(&jrp->outlock);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
|
|
|
/* Finally, execute user's callback */
|
|
|
|
usercall(dev, userdesc, userstatus, userarg);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* reenable / unmask IRQs */
|
|
|
|
clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
|
|
|
|
}
|
|
|
|
|
2013-10-25 14:31:02 +08:00
|
|
|
/**
|
|
|
|
* caam_jr_alloc() - Alloc a job ring for someone to use as needed.
|
|
|
|
*
|
|
|
|
* returns : pointer to the newly allocated physical
|
|
|
|
* JobR dev can be written to if successful.
|
|
|
|
**/
|
|
|
|
struct device *caam_jr_alloc(void)
|
|
|
|
{
|
|
|
|
struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
|
|
|
|
struct device *dev = NULL;
|
|
|
|
int min_tfm_cnt = INT_MAX;
|
|
|
|
int tfm_cnt;
|
|
|
|
|
|
|
|
spin_lock(&driver_data.jr_alloc_lock);
|
|
|
|
|
|
|
|
if (list_empty(&driver_data.jr_list)) {
|
|
|
|
spin_unlock(&driver_data.jr_alloc_lock);
|
|
|
|
return ERR_PTR(-ENODEV);
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
|
|
|
|
tfm_cnt = atomic_read(&jrpriv->tfm_count);
|
|
|
|
if (tfm_cnt < min_tfm_cnt) {
|
|
|
|
min_tfm_cnt = tfm_cnt;
|
|
|
|
min_jrpriv = jrpriv;
|
|
|
|
}
|
|
|
|
if (!min_tfm_cnt)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (min_jrpriv) {
|
|
|
|
atomic_inc(&min_jrpriv->tfm_count);
|
|
|
|
dev = min_jrpriv->dev;
|
|
|
|
}
|
|
|
|
spin_unlock(&driver_data.jr_alloc_lock);
|
|
|
|
|
|
|
|
return dev;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(caam_jr_alloc);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* caam_jr_free() - Free the Job Ring
|
|
|
|
* @rdev - points to the dev that identifies the Job ring to
|
|
|
|
* be released.
|
|
|
|
**/
|
|
|
|
void caam_jr_free(struct device *rdev)
|
|
|
|
{
|
|
|
|
struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
|
|
|
|
|
|
|
|
atomic_dec(&jrpriv->tfm_count);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(caam_jr_free);
|
|
|
|
|
2011-03-13 16:54:26 +08:00
|
|
|
/**
|
|
|
|
* caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
|
|
|
|
* -EBUSY if the queue is full, -EIO if it cannot map the caller's
|
|
|
|
* descriptor.
|
|
|
|
* @dev: device of the job ring to be used. This device should have
|
|
|
|
* been assigned prior by caam_jr_register().
|
|
|
|
* @desc: points to a job descriptor that execute our request. All
|
|
|
|
* descriptors (and all referenced data) must be in a DMAable
|
|
|
|
* region, and all data references must be physical addresses
|
|
|
|
* accessible to CAAM (i.e. within a PAMU window granted
|
|
|
|
* to it).
|
|
|
|
* @cbk: pointer to a callback function to be invoked upon completion
|
|
|
|
* of this request. This has the form:
|
|
|
|
* callback(struct device *dev, u32 *desc, u32 stat, void *arg)
|
|
|
|
* where:
|
|
|
|
* @dev: contains the job ring device that processed this
|
|
|
|
* response.
|
|
|
|
* @desc: descriptor that initiated the request, same as
|
|
|
|
* "desc" being argued to caam_jr_enqueue().
|
|
|
|
* @status: untranslated status received from CAAM. See the
|
|
|
|
* reference manual for a detailed description of
|
|
|
|
* error meaning, or see the JRSTA definitions in the
|
|
|
|
* register header file
|
|
|
|
* @areq: optional pointer to an argument passed with the
|
|
|
|
* original request
|
|
|
|
* @areq: optional pointer to a user argument for use at callback
|
|
|
|
* time.
|
|
|
|
**/
|
|
|
|
int caam_jr_enqueue(struct device *dev, u32 *desc,
|
|
|
|
void (*cbk)(struct device *dev, u32 *desc,
|
|
|
|
u32 status, void *areq),
|
|
|
|
void *areq)
|
|
|
|
{
|
|
|
|
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
|
|
|
|
struct caam_jrentry_info *head_entry;
|
|
|
|
int head, tail, desc_size;
|
|
|
|
dma_addr_t desc_dma;
|
|
|
|
|
|
|
|
desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
|
|
|
|
desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
|
|
|
|
if (dma_mapping_error(dev, desc_dma)) {
|
|
|
|
dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
crypto: caam - fix possible deadlock condition
commit "crypto: caam - use non-irq versions of spinlocks for job rings"
made two bad assumptions:
(a) The caam_jr_enqueue lock isn't used in softirq context.
Not true: jr_enqueue can be interrupted by an incoming net
interrupt and the received packet may be sent for encryption,
via caam_jr_enqueue in softirq context, thereby inducing a
deadlock.
This is evidenced when running netperf over an IPSec tunnel
between two P4080's, with spinlock debugging turned on:
[ 892.092569] BUG: spinlock lockup on CPU#7, netperf/10634, e8bf5f70
[ 892.098747] Call Trace:
[ 892.101197] [eff9fc10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.107563] [eff9fc50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.113399] [eff9fc80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.118889] [eff9fc90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.124550] [eff9fcd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129625] BUG: spinlock lockup on CPU#5, swapper/5/0, e8bf5f70
[ 892.129629] Call Trace:
[ 892.129637] [effa7c10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.129645] [effa7c50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.129652] [effa7c80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.129660] [effa7c90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.129666] [effa7cd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129674] [effa7d00] [c0509724] esp_input+0x178/0x334
[ 892.129681] [effa7d50] [c0519778] xfrm_input+0x77c/0x818
[ 892.129688] [effa7da0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.129697] [effa7db0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.129703] [effa7de0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.129709] [effa7e10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.129715] [effa7e80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.129726] [effa7eb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.129732] [effa7ef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.129742] [effa7f10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.129748] [effa7f40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.129754] [effa7f60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.129763] [effa7fa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.129769] [effa7ff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.129775] [ebacfe70] [c0004868] do_softirq+0xd8/0x104
[ 892.129780] [ebacfe90] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.129786] [ebacfea0] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.129792] [ebacfed0] [c000fad8] ret_from_except+0x0/0x18
[ 892.129798] [ebacff90] [c0009010] cpu_idle+0x94/0xf0
[ 892.129804] [ebacffb0] [c059ff88] start_secondary+0x42c/0x430
[ 892.129809] [ebacfff0] [c0001e28] __secondary_start+0x30/0x84
[ 892.281474]
[ 892.282959] [eff9fd00] [c0509724] esp_input+0x178/0x334
[ 892.288186] [eff9fd50] [c0519778] xfrm_input+0x77c/0x818
[ 892.293499] [eff9fda0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.299074] [eff9fdb0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.304907] [eff9fde0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.309872] [eff9fe10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.315966] [eff9fe80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.321803] [eff9feb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.326855] [eff9fef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.333212] [eff9ff10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.338872] [eff9ff40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.344271] [eff9ff60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.349846] [eff9ffa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.355338] [eff9fff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.360910] [e7169950] [c0004868] do_softirq+0xd8/0x104
[ 892.366135] [e7169970] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.371101] [e7169980] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.375979] [e71699b0] [c000fad8] ret_from_except+0x0/0x18
[ 892.381466] [e7169a70] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.387127] [e7169ab0] [c044ad4c] aead_givencrypt+0x6ac/0xa70
[ 892.392873] [e7169b20] [c050a0b8] esp_output+0x2b4/0x570
[ 892.398186] [e7169b80] [c0519b9c] xfrm_output_resume+0x248/0x7c0
[ 892.404194] [e7169bb0] [c050e89c] xfrm4_output_finish+0x18/0x28
[ 892.410113] [e7169bc0] [c050e8f4] xfrm4_output+0x48/0x98
[ 892.415427] [e7169bd0] [c04beac0] ip_local_out+0x48/0x98
[ 892.420740] [e7169be0] [c04bec7c] ip_queue_xmit+0x16c/0x490
[ 892.426314] [e7169c10] [c04d6128] tcp_transmit_skb+0x35c/0x9a4
[ 892.432147] [e7169c70] [c04d6f98] tcp_write_xmit+0x200/0xa04
[ 892.437808] [e7169cc0] [c04c8ccc] tcp_sendmsg+0x994/0xcec
[ 892.443213] [e7169d40] [c04eebfc] inet_sendmsg+0xd0/0x164
[ 892.448617] [e7169d70] [c04792f8] sock_sendmsg+0x8c/0xbc
[ 892.453931] [e7169e40] [c047aecc] sys_sendto+0xc0/0xfc
[ 892.459069] [e7169f10] [c047b934] sys_socketcall+0x110/0x25c
[ 892.464729] [e7169f40] [c000f480] ret_from_syscall+0x0/0x3c
(b) since the caam_jr_dequeue lock is only used in bh context,
then semantically it should use _bh spin_lock types. spin_lock_bh
semantics are to disable back-halves, and used when a lock is shared
between softirq (bh) context and process and/or h/w IRQ context.
Since the lock is only used within softirq context, and this tasklet
is atomic, there is no need to do the additional work to disable
back halves.
This patch adds back-half disabling protection to caam_jr_enqueue
spin_locks to fix (a), and drops it from caam_jr_dequeue to fix (b).
Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2012-07-14 07:04:23 +08:00
|
|
|
spin_lock_bh(&jrp->inplock);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
|
|
|
head = jrp->head;
|
|
|
|
tail = ACCESS_ONCE(jrp->tail);
|
|
|
|
|
|
|
|
if (!rd_reg32(&jrp->rregs->inpring_avail) ||
|
|
|
|
CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
|
crypto: caam - fix possible deadlock condition
commit "crypto: caam - use non-irq versions of spinlocks for job rings"
made two bad assumptions:
(a) The caam_jr_enqueue lock isn't used in softirq context.
Not true: jr_enqueue can be interrupted by an incoming net
interrupt and the received packet may be sent for encryption,
via caam_jr_enqueue in softirq context, thereby inducing a
deadlock.
This is evidenced when running netperf over an IPSec tunnel
between two P4080's, with spinlock debugging turned on:
[ 892.092569] BUG: spinlock lockup on CPU#7, netperf/10634, e8bf5f70
[ 892.098747] Call Trace:
[ 892.101197] [eff9fc10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.107563] [eff9fc50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.113399] [eff9fc80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.118889] [eff9fc90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.124550] [eff9fcd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129625] BUG: spinlock lockup on CPU#5, swapper/5/0, e8bf5f70
[ 892.129629] Call Trace:
[ 892.129637] [effa7c10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.129645] [effa7c50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.129652] [effa7c80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.129660] [effa7c90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.129666] [effa7cd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129674] [effa7d00] [c0509724] esp_input+0x178/0x334
[ 892.129681] [effa7d50] [c0519778] xfrm_input+0x77c/0x818
[ 892.129688] [effa7da0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.129697] [effa7db0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.129703] [effa7de0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.129709] [effa7e10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.129715] [effa7e80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.129726] [effa7eb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.129732] [effa7ef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.129742] [effa7f10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.129748] [effa7f40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.129754] [effa7f60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.129763] [effa7fa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.129769] [effa7ff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.129775] [ebacfe70] [c0004868] do_softirq+0xd8/0x104
[ 892.129780] [ebacfe90] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.129786] [ebacfea0] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.129792] [ebacfed0] [c000fad8] ret_from_except+0x0/0x18
[ 892.129798] [ebacff90] [c0009010] cpu_idle+0x94/0xf0
[ 892.129804] [ebacffb0] [c059ff88] start_secondary+0x42c/0x430
[ 892.129809] [ebacfff0] [c0001e28] __secondary_start+0x30/0x84
[ 892.281474]
[ 892.282959] [eff9fd00] [c0509724] esp_input+0x178/0x334
[ 892.288186] [eff9fd50] [c0519778] xfrm_input+0x77c/0x818
[ 892.293499] [eff9fda0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.299074] [eff9fdb0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.304907] [eff9fde0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.309872] [eff9fe10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.315966] [eff9fe80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.321803] [eff9feb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.326855] [eff9fef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.333212] [eff9ff10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.338872] [eff9ff40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.344271] [eff9ff60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.349846] [eff9ffa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.355338] [eff9fff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.360910] [e7169950] [c0004868] do_softirq+0xd8/0x104
[ 892.366135] [e7169970] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.371101] [e7169980] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.375979] [e71699b0] [c000fad8] ret_from_except+0x0/0x18
[ 892.381466] [e7169a70] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.387127] [e7169ab0] [c044ad4c] aead_givencrypt+0x6ac/0xa70
[ 892.392873] [e7169b20] [c050a0b8] esp_output+0x2b4/0x570
[ 892.398186] [e7169b80] [c0519b9c] xfrm_output_resume+0x248/0x7c0
[ 892.404194] [e7169bb0] [c050e89c] xfrm4_output_finish+0x18/0x28
[ 892.410113] [e7169bc0] [c050e8f4] xfrm4_output+0x48/0x98
[ 892.415427] [e7169bd0] [c04beac0] ip_local_out+0x48/0x98
[ 892.420740] [e7169be0] [c04bec7c] ip_queue_xmit+0x16c/0x490
[ 892.426314] [e7169c10] [c04d6128] tcp_transmit_skb+0x35c/0x9a4
[ 892.432147] [e7169c70] [c04d6f98] tcp_write_xmit+0x200/0xa04
[ 892.437808] [e7169cc0] [c04c8ccc] tcp_sendmsg+0x994/0xcec
[ 892.443213] [e7169d40] [c04eebfc] inet_sendmsg+0xd0/0x164
[ 892.448617] [e7169d70] [c04792f8] sock_sendmsg+0x8c/0xbc
[ 892.453931] [e7169e40] [c047aecc] sys_sendto+0xc0/0xfc
[ 892.459069] [e7169f10] [c047b934] sys_socketcall+0x110/0x25c
[ 892.464729] [e7169f40] [c000f480] ret_from_syscall+0x0/0x3c
(b) since the caam_jr_dequeue lock is only used in bh context,
then semantically it should use _bh spin_lock types. spin_lock_bh
semantics are to disable back-halves, and used when a lock is shared
between softirq (bh) context and process and/or h/w IRQ context.
Since the lock is only used within softirq context, and this tasklet
is atomic, there is no need to do the additional work to disable
back halves.
This patch adds back-half disabling protection to caam_jr_enqueue
spin_locks to fix (a), and drops it from caam_jr_dequeue to fix (b).
Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2012-07-14 07:04:23 +08:00
|
|
|
spin_unlock_bh(&jrp->inplock);
|
2011-03-13 16:54:26 +08:00
|
|
|
dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
|
|
|
|
|
|
|
head_entry = &jrp->entinfo[head];
|
|
|
|
head_entry->desc_addr_virt = desc;
|
|
|
|
head_entry->desc_size = desc_size;
|
|
|
|
head_entry->callbk = (void *)cbk;
|
|
|
|
head_entry->cbkarg = areq;
|
|
|
|
head_entry->desc_addr_dma = desc_dma;
|
|
|
|
|
|
|
|
jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
|
|
|
|
|
2015-08-06 02:28:35 +08:00
|
|
|
/*
|
|
|
|
* Guarantee that the descriptor's DMA address has been written to
|
|
|
|
* the next slot in the ring before the write index is updated, since
|
|
|
|
* other cores may update this index independently.
|
|
|
|
*/
|
2011-03-13 16:54:26 +08:00
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
|
|
|
|
(JOBR_DEPTH - 1);
|
|
|
|
jrp->head = (head + 1) & (JOBR_DEPTH - 1);
|
|
|
|
|
2015-08-06 02:28:35 +08:00
|
|
|
/*
|
|
|
|
* Ensure that all job information has been written before
|
|
|
|
* notifying CAAM that a new job was added to the input ring.
|
|
|
|
*/
|
|
|
|
wmb();
|
|
|
|
|
2011-03-13 16:54:26 +08:00
|
|
|
wr_reg32(&jrp->rregs->inpring_jobadd, 1);
|
|
|
|
|
crypto: caam - fix possible deadlock condition
commit "crypto: caam - use non-irq versions of spinlocks for job rings"
made two bad assumptions:
(a) The caam_jr_enqueue lock isn't used in softirq context.
Not true: jr_enqueue can be interrupted by an incoming net
interrupt and the received packet may be sent for encryption,
via caam_jr_enqueue in softirq context, thereby inducing a
deadlock.
This is evidenced when running netperf over an IPSec tunnel
between two P4080's, with spinlock debugging turned on:
[ 892.092569] BUG: spinlock lockup on CPU#7, netperf/10634, e8bf5f70
[ 892.098747] Call Trace:
[ 892.101197] [eff9fc10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.107563] [eff9fc50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.113399] [eff9fc80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.118889] [eff9fc90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.124550] [eff9fcd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129625] BUG: spinlock lockup on CPU#5, swapper/5/0, e8bf5f70
[ 892.129629] Call Trace:
[ 892.129637] [effa7c10] [c00084c0] show_stack+0x48/0x15c (unreliable)
[ 892.129645] [effa7c50] [c0239c2c] do_raw_spin_lock+0x16c/0x174
[ 892.129652] [effa7c80] [c0596494] _raw_spin_lock+0x3c/0x50
[ 892.129660] [effa7c90] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.129666] [effa7cd0] [c044a644] aead_decrypt+0x6c/0xc8
[ 892.129674] [effa7d00] [c0509724] esp_input+0x178/0x334
[ 892.129681] [effa7d50] [c0519778] xfrm_input+0x77c/0x818
[ 892.129688] [effa7da0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.129697] [effa7db0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.129703] [effa7de0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.129709] [effa7e10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.129715] [effa7e80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.129726] [effa7eb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.129732] [effa7ef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.129742] [effa7f10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.129748] [effa7f40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.129754] [effa7f60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.129763] [effa7fa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.129769] [effa7ff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.129775] [ebacfe70] [c0004868] do_softirq+0xd8/0x104
[ 892.129780] [ebacfe90] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.129786] [ebacfea0] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.129792] [ebacfed0] [c000fad8] ret_from_except+0x0/0x18
[ 892.129798] [ebacff90] [c0009010] cpu_idle+0x94/0xf0
[ 892.129804] [ebacffb0] [c059ff88] start_secondary+0x42c/0x430
[ 892.129809] [ebacfff0] [c0001e28] __secondary_start+0x30/0x84
[ 892.281474]
[ 892.282959] [eff9fd00] [c0509724] esp_input+0x178/0x334
[ 892.288186] [eff9fd50] [c0519778] xfrm_input+0x77c/0x818
[ 892.293499] [eff9fda0] [c050e344] xfrm4_rcv_encap+0x20/0x30
[ 892.299074] [eff9fdb0] [c04b90c8] ip_local_deliver+0x190/0x408
[ 892.304907] [eff9fde0] [c04b966c] ip_rcv+0x32c/0x898
[ 892.309872] [eff9fe10] [c048b998] __netif_receive_skb+0x27c/0x4e8
[ 892.315966] [eff9fe80] [c048d744] netif_receive_skb+0x4c/0x13c
[ 892.321803] [eff9feb0] [c03c28ac] _dpa_rx+0x1a8/0x354
[ 892.326855] [eff9fef0] [c03c2ac4] ingress_rx_default_dqrr+0x6c/0x108
[ 892.333212] [eff9ff10] [c0467ae0] qman_poll_dqrr+0x170/0x1d4
[ 892.338872] [eff9ff40] [c03c153c] dpaa_eth_poll+0x20/0x94
[ 892.344271] [eff9ff60] [c048dbd0] net_rx_action+0x13c/0x1f4
[ 892.349846] [eff9ffa0] [c003d1b8] __do_softirq+0x108/0x1b0
[ 892.355338] [eff9fff0] [c000df58] call_do_softirq+0x14/0x24
[ 892.360910] [e7169950] [c0004868] do_softirq+0xd8/0x104
[ 892.366135] [e7169970] [c003d5a4] irq_exit+0xb8/0xd8
[ 892.371101] [e7169980] [c0004498] do_IRQ+0xa4/0x1b0
[ 892.375979] [e71699b0] [c000fad8] ret_from_except+0x0/0x18
[ 892.381466] [e7169a70] [c0445e74] caam_jr_enqueue+0xf8/0x250
[ 892.387127] [e7169ab0] [c044ad4c] aead_givencrypt+0x6ac/0xa70
[ 892.392873] [e7169b20] [c050a0b8] esp_output+0x2b4/0x570
[ 892.398186] [e7169b80] [c0519b9c] xfrm_output_resume+0x248/0x7c0
[ 892.404194] [e7169bb0] [c050e89c] xfrm4_output_finish+0x18/0x28
[ 892.410113] [e7169bc0] [c050e8f4] xfrm4_output+0x48/0x98
[ 892.415427] [e7169bd0] [c04beac0] ip_local_out+0x48/0x98
[ 892.420740] [e7169be0] [c04bec7c] ip_queue_xmit+0x16c/0x490
[ 892.426314] [e7169c10] [c04d6128] tcp_transmit_skb+0x35c/0x9a4
[ 892.432147] [e7169c70] [c04d6f98] tcp_write_xmit+0x200/0xa04
[ 892.437808] [e7169cc0] [c04c8ccc] tcp_sendmsg+0x994/0xcec
[ 892.443213] [e7169d40] [c04eebfc] inet_sendmsg+0xd0/0x164
[ 892.448617] [e7169d70] [c04792f8] sock_sendmsg+0x8c/0xbc
[ 892.453931] [e7169e40] [c047aecc] sys_sendto+0xc0/0xfc
[ 892.459069] [e7169f10] [c047b934] sys_socketcall+0x110/0x25c
[ 892.464729] [e7169f40] [c000f480] ret_from_syscall+0x0/0x3c
(b) since the caam_jr_dequeue lock is only used in bh context,
then semantically it should use _bh spin_lock types. spin_lock_bh
semantics are to disable back-halves, and used when a lock is shared
between softirq (bh) context and process and/or h/w IRQ context.
Since the lock is only used within softirq context, and this tasklet
is atomic, there is no need to do the additional work to disable
back halves.
This patch adds back-half disabling protection to caam_jr_enqueue
spin_locks to fix (a), and drops it from caam_jr_dequeue to fix (b).
Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2012-07-14 07:04:23 +08:00
|
|
|
spin_unlock_bh(&jrp->inplock);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(caam_jr_enqueue);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Init JobR independent of platform property detection
|
|
|
|
*/
|
|
|
|
static int caam_jr_init(struct device *dev)
|
|
|
|
{
|
|
|
|
struct caam_drv_private_jr *jrp;
|
|
|
|
dma_addr_t inpbusaddr, outbusaddr;
|
|
|
|
int i, error;
|
|
|
|
|
|
|
|
jrp = dev_get_drvdata(dev);
|
|
|
|
|
2012-06-23 08:48:57 +08:00
|
|
|
tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
|
2011-04-12 08:15:16 +08:00
|
|
|
|
2012-06-23 08:48:57 +08:00
|
|
|
/* Connect job ring interrupt handler. */
|
2011-04-12 08:15:16 +08:00
|
|
|
error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
|
2013-10-25 14:31:01 +08:00
|
|
|
dev_name(dev), dev);
|
2011-04-12 08:15:16 +08:00
|
|
|
if (error) {
|
|
|
|
dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
|
|
|
|
jrp->ridx, jrp->irq);
|
2015-01-22 22:00:49 +08:00
|
|
|
goto out_kill_deq;
|
2011-04-12 08:15:16 +08:00
|
|
|
}
|
|
|
|
|
2011-03-13 16:54:26 +08:00
|
|
|
error = caam_reset_hw_jr(dev);
|
|
|
|
if (error)
|
2015-01-22 22:00:49 +08:00
|
|
|
goto out_free_irq;
|
2011-03-13 16:54:26 +08:00
|
|
|
|
2015-01-22 22:00:49 +08:00
|
|
|
error = -ENOMEM;
|
2015-08-22 00:52:00 +08:00
|
|
|
jrp->inpring = dma_alloc_coherent(dev, sizeof(*jrp->inpring) *
|
|
|
|
JOBR_DEPTH, &inpbusaddr, GFP_KERNEL);
|
2015-01-22 22:00:49 +08:00
|
|
|
if (!jrp->inpring)
|
|
|
|
goto out_free_irq;
|
2012-07-11 11:06:10 +08:00
|
|
|
|
2015-08-22 00:52:00 +08:00
|
|
|
jrp->outring = dma_alloc_coherent(dev, sizeof(*jrp->outring) *
|
2012-07-11 11:06:10 +08:00
|
|
|
JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
|
2015-01-22 22:00:49 +08:00
|
|
|
if (!jrp->outring)
|
|
|
|
goto out_free_inpring;
|
2011-03-13 16:54:26 +08:00
|
|
|
|
2015-08-22 00:52:00 +08:00
|
|
|
jrp->entinfo = kcalloc(JOBR_DEPTH, sizeof(*jrp->entinfo), GFP_KERNEL);
|
2015-01-22 22:00:49 +08:00
|
|
|
if (!jrp->entinfo)
|
|
|
|
goto out_free_outring;
|
2011-03-13 16:54:26 +08:00
|
|
|
|
|
|
|
for (i = 0; i < JOBR_DEPTH; i++)
|
|
|
|
jrp->entinfo[i].desc_addr_dma = !0;
|
|
|
|
|
|
|
|
/* Setup rings */
|
|
|
|
jrp->inp_ring_write_index = 0;
|
|
|
|
jrp->out_ring_read_index = 0;
|
|
|
|
jrp->head = 0;
|
|
|
|
jrp->tail = 0;
|
|
|
|
|
|
|
|
wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
|
|
|
|
wr_reg64(&jrp->rregs->outring_base, outbusaddr);
|
|
|
|
wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
|
|
|
|
wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
|
|
|
|
|
|
|
|
jrp->ringsize = JOBR_DEPTH;
|
|
|
|
|
|
|
|
spin_lock_init(&jrp->inplock);
|
|
|
|
spin_lock_init(&jrp->outlock);
|
|
|
|
|
|
|
|
/* Select interrupt coalescing parameters */
|
|
|
|
setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
|
|
|
|
(JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
|
|
|
|
(JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
|
|
|
|
|
|
|
|
return 0;
|
2015-01-22 22:00:49 +08:00
|
|
|
|
|
|
|
out_free_outring:
|
|
|
|
dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
|
|
|
|
jrp->outring, outbusaddr);
|
|
|
|
out_free_inpring:
|
|
|
|
dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
|
|
|
|
jrp->inpring, inpbusaddr);
|
|
|
|
dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
|
|
|
|
out_free_irq:
|
|
|
|
free_irq(jrp->irq, dev);
|
|
|
|
out_kill_deq:
|
|
|
|
tasklet_kill(&jrp->irqtask);
|
|
|
|
return error;
|
2011-03-13 16:54:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
2013-10-25 14:31:01 +08:00
|
|
|
* Probe routine for each detected JobR subsystem.
|
2011-03-13 16:54:26 +08:00
|
|
|
*/
|
2013-10-25 14:31:01 +08:00
|
|
|
static int caam_jr_probe(struct platform_device *pdev)
|
2011-03-13 16:54:26 +08:00
|
|
|
{
|
2013-10-25 14:31:01 +08:00
|
|
|
struct device *jrdev;
|
|
|
|
struct device_node *nprop;
|
|
|
|
struct caam_job_ring __iomem *ctrl;
|
2011-03-13 16:54:26 +08:00
|
|
|
struct caam_drv_private_jr *jrpriv;
|
2013-10-25 14:31:01 +08:00
|
|
|
static int total_jobrs;
|
2011-03-13 16:54:26 +08:00
|
|
|
int error;
|
|
|
|
|
2013-10-25 14:31:01 +08:00
|
|
|
jrdev = &pdev->dev;
|
2015-08-22 00:52:00 +08:00
|
|
|
jrpriv = devm_kmalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL);
|
2013-10-25 14:31:01 +08:00
|
|
|
if (!jrpriv)
|
2011-03-13 16:54:26 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2013-10-25 14:31:01 +08:00
|
|
|
dev_set_drvdata(jrdev, jrpriv);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
2013-10-25 14:31:01 +08:00
|
|
|
/* save ring identity relative to detection */
|
|
|
|
jrpriv->ridx = total_jobrs++;
|
|
|
|
|
|
|
|
nprop = pdev->dev.of_node;
|
|
|
|
/* Get configuration properties from device tree */
|
|
|
|
/* First, get register page */
|
|
|
|
ctrl = of_iomap(nprop, 0);
|
|
|
|
if (!ctrl) {
|
|
|
|
dev_err(jrdev, "of_iomap() failed\n");
|
|
|
|
return -ENOMEM;
|
2011-03-13 16:54:26 +08:00
|
|
|
}
|
2013-04-15 12:25:51 +08:00
|
|
|
|
2013-10-25 14:31:01 +08:00
|
|
|
jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
|
2011-03-13 16:54:26 +08:00
|
|
|
|
2012-06-23 08:48:51 +08:00
|
|
|
if (sizeof(dma_addr_t) == sizeof(u64))
|
2013-10-25 14:31:01 +08:00
|
|
|
if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
|
2014-07-11 20:34:47 +08:00
|
|
|
dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40));
|
2012-06-23 08:48:51 +08:00
|
|
|
else
|
2014-07-11 20:34:47 +08:00
|
|
|
dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36));
|
2012-06-23 08:48:51 +08:00
|
|
|
else
|
2014-07-11 20:34:47 +08:00
|
|
|
dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
|
2012-06-23 08:48:51 +08:00
|
|
|
|
2011-03-13 16:54:26 +08:00
|
|
|
/* Identify the interrupt */
|
2013-11-24 08:18:25 +08:00
|
|
|
jrpriv->irq = irq_of_parse_and_map(nprop, 0);
|
2011-03-13 16:54:26 +08:00
|
|
|
|
|
|
|
/* Now do the platform independent part */
|
|
|
|
error = caam_jr_init(jrdev); /* now turn on hardware */
|
2015-01-22 22:00:48 +08:00
|
|
|
if (error) {
|
|
|
|
irq_dispose_mapping(jrpriv->irq);
|
2011-03-13 16:54:26 +08:00
|
|
|
return error;
|
2015-01-22 22:00:48 +08:00
|
|
|
}
|
2011-03-13 16:54:26 +08:00
|
|
|
|
2013-10-25 14:31:01 +08:00
|
|
|
jrpriv->dev = jrdev;
|
|
|
|
spin_lock(&driver_data.jr_alloc_lock);
|
|
|
|
list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
|
|
|
|
spin_unlock(&driver_data.jr_alloc_lock);
|
|
|
|
|
2013-10-25 14:31:02 +08:00
|
|
|
atomic_set(&jrpriv->tfm_count, 0);
|
|
|
|
|
2013-10-25 14:31:01 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct of_device_id caam_jr_match[] = {
|
|
|
|
{
|
|
|
|
.compatible = "fsl,sec-v4.0-job-ring",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.compatible = "fsl,sec4.0-job-ring",
|
|
|
|
},
|
|
|
|
{},
|
|
|
|
};
|
|
|
|
MODULE_DEVICE_TABLE(of, caam_jr_match);
|
|
|
|
|
|
|
|
static struct platform_driver caam_jr_driver = {
|
|
|
|
.driver = {
|
|
|
|
.name = "caam_jr",
|
|
|
|
.of_match_table = caam_jr_match,
|
|
|
|
},
|
|
|
|
.probe = caam_jr_probe,
|
|
|
|
.remove = caam_jr_remove,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int __init jr_driver_init(void)
|
|
|
|
{
|
|
|
|
spin_lock_init(&driver_data.jr_alloc_lock);
|
|
|
|
INIT_LIST_HEAD(&driver_data.jr_list);
|
|
|
|
return platform_driver_register(&caam_jr_driver);
|
2011-03-13 16:54:26 +08:00
|
|
|
}
|
2013-10-25 14:31:01 +08:00
|
|
|
|
|
|
|
static void __exit jr_driver_exit(void)
|
|
|
|
{
|
|
|
|
platform_driver_unregister(&caam_jr_driver);
|
2011-03-13 16:54:26 +08:00
|
|
|
}
|
2013-10-25 14:31:01 +08:00
|
|
|
|
|
|
|
module_init(jr_driver_init);
|
|
|
|
module_exit(jr_driver_exit);
|
|
|
|
|
|
|
|
MODULE_LICENSE("GPL");
|
|
|
|
MODULE_DESCRIPTION("FSL CAAM JR request backend");
|
|
|
|
MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
|