openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

dmaengine: ioatdma: clean up local dma channel data structure 

Kill the common ioatdma channel structure and everything that is not
dma_chan to be ioat_dma_chan. Since we don't have to worry about v1
and v2 ioatdma anymore this makes it much cleaner and obvious for
maintenance.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Acked-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
This commit is contained in:
Dave Jiang 2015-08-11 08:48:21 -07:00 committed by Vinod Koul
parent 7f832645d0
commit 5a976888c9
5 changed files with 469 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
drivers/dma/ioat/dma.c
View File

@ -50,7 +50,7 @@ MODULE_PARM_DESC(ioat_pending_level,
static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
{ {
struct ioatdma_device *instance = data; struct ioatdma_device *instance = data;
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
unsigned long attnstatus; unsigned long attnstatus;
int bit; int bit;
u8 intrctrl; u8 intrctrl;
@ -67,9 +67,9 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
chan = ioat_chan_by_index(instance, bit); ioat_chan = ioat_chan_by_index(instance, bit);
if (test_bit(IOAT_RUN, &chan->state)) if (test_bit(IOAT_RUN, &ioat_chan->state))
tasklet_schedule(&chan->cleanup_task); tasklet_schedule(&ioat_chan->cleanup_task);
} }
writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
@ -83,45 +83,47 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
*/ */
static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
{ {
struct ioat_chan_common *chan = data; struct ioatdma_chan *ioat_chan = data;
if (test_bit(IOAT_RUN, &chan->state)) if (test_bit(IOAT_RUN, &ioat_chan->state))
tasklet_schedule(&chan->cleanup_task); tasklet_schedule(&ioat_chan->cleanup_task);
return IRQ_HANDLED; return IRQ_HANDLED;
} }
/* common channel initialization */ /* common channel initialization */
void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) void
ioat_init_channel(struct ioatdma_device *device, struct ioatdma_chan *ioat_chan,
int idx)
{ {
struct dma_device *dma = &device->common; struct dma_device *dma = &device->common;
struct dma_chan *c = &chan->common; struct dma_chan *c = &ioat_chan->dma_chan;
unsigned long data = (unsigned long) c; unsigned long data = (unsigned long) c;
chan->device = device; ioat_chan->device = device;
chan->reg_base = device->reg_base + (0x80 * (idx + 1)); ioat_chan->reg_base = device->reg_base + (0x80 * (idx + 1));
spin_lock_init(&chan->cleanup_lock); spin_lock_init(&ioat_chan->cleanup_lock);
chan->common.device = dma; ioat_chan->dma_chan.device = dma;
dma_cookie_init(&chan->common); dma_cookie_init(&ioat_chan->dma_chan);
list_add_tail(&chan->common.device_node, &dma->channels); list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
device->idx[idx] = chan; device->idx[idx] = ioat_chan;
init_timer(&chan->timer); init_timer(&ioat_chan->timer);
chan->timer.function = device->timer_fn; ioat_chan->timer.function = device->timer_fn;
chan->timer.data = data; ioat_chan->timer.data = data;
tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); tasklet_init(&ioat_chan->cleanup_task, device->cleanup_fn, data);
} }
void ioat_stop(struct ioat_chan_common *chan) void ioat_stop(struct ioatdma_chan *ioat_chan)
{ {
struct ioatdma_device *device = chan->device; struct ioatdma_device *device = ioat_chan->device;
struct pci_dev *pdev = device->pdev; struct pci_dev *pdev = device->pdev;
int chan_id = chan_num(chan); int chan_id = chan_num(ioat_chan);
struct msix_entry *msix; struct msix_entry *msix;
/* 1/ stop irq from firing tasklets /* 1/ stop irq from firing tasklets
* 2/ stop the tasklet from re-arming irqs * 2/ stop the tasklet from re-arming irqs
*/ */
clear_bit(IOAT_RUN, &chan->state); clear_bit(IOAT_RUN, &ioat_chan->state);
/* flush inflight interrupts */ /* flush inflight interrupts */
switch (device->irq_mode) { switch (device->irq_mode) {
@ -138,29 +140,30 @@ void ioat_stop(struct ioat_chan_common *chan)
} }
/* flush inflight timers */ /* flush inflight timers */
del_timer_sync(&chan->timer); del_timer_sync(&ioat_chan->timer);
/* flush inflight tasklet runs */ /* flush inflight tasklet runs */
tasklet_kill(&chan->cleanup_task); tasklet_kill(&ioat_chan->cleanup_task);
/* final cleanup now that everything is quiesced and can't re-arm */ /* final cleanup now that everything is quiesced and can't re-arm */
device->cleanup_fn((unsigned long) &chan->common); device->cleanup_fn((unsigned long)&ioat_chan->dma_chan);
} }
dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) dma_addr_t ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
{ {
dma_addr_t phys_complete; dma_addr_t phys_complete;
u64 completion; u64 completion;
completion = *chan->completion; completion = *ioat_chan->completion;
phys_complete = ioat_chansts_to_addr(completion); phys_complete = ioat_chansts_to_addr(completion);
dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
(unsigned long long) phys_complete); (unsigned long long) phys_complete);
if (is_ioat_halted(completion)) { if (is_ioat_halted(completion)) {
u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n",
chanerr); chanerr);
/* TODO do something to salvage the situation */ /* TODO do something to salvage the situation */
@ -169,14 +172,14 @@ dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan)
return phys_complete; return phys_complete;
} }
bool ioat_cleanup_preamble(struct ioat_chan_common *chan, bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
dma_addr_t *phys_complete) dma_addr_t *phys_complete)
{ {
*phys_complete = ioat_get_current_completion(chan); *phys_complete = ioat_get_current_completion(ioat_chan);
if (*phys_complete == chan->last_completion) if (*phys_complete == ioat_chan->last_completion)
return false; return false;
clear_bit(IOAT_COMPLETION_ACK, &chan->state); clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
return true; return true;
} }
@ -185,8 +188,8 @@ enum dma_status
ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
struct dma_tx_state *txstate) struct dma_tx_state *txstate)
{ {
struct ioat_chan_common *chan = to_chan_common(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioatdma_device *device = chan->device; struct ioatdma_device *device = ioat_chan->device;
enum dma_status ret; enum dma_status ret;
ret = dma_cookie_status(c, cookie, txstate); ret = dma_cookie_status(c, cookie, txstate);
@ -322,7 +325,7 @@ MODULE_PARM_DESC(ioat_interrupt_style,
*/ */
int ioat_dma_setup_interrupts(struct ioatdma_device *device) int ioat_dma_setup_interrupts(struct ioatdma_device *device)
{ {
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
struct pci_dev *pdev = device->pdev; struct pci_dev *pdev = device->pdev;
struct device *dev = &pdev->dev; struct device *dev = &pdev->dev;
struct msix_entry *msix; struct msix_entry *msix;
@ -351,15 +354,15 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *device)
for (i = 0; i < msixcnt; i++) { for (i = 0; i < msixcnt; i++) {
msix = &device->msix_entries[i]; msix = &device->msix_entries[i];
chan = ioat_chan_by_index(device, i); ioat_chan = ioat_chan_by_index(device, i);
err = devm_request_irq(dev, msix->vector, err = devm_request_irq(dev, msix->vector,
ioat_dma_do_interrupt_msix, 0, ioat_dma_do_interrupt_msix, 0,
"ioat-msix", chan); "ioat-msix", ioat_chan);
if (err) { if (err) {
for (j = 0; j < i; j++) { for (j = 0; j < i; j++) {
msix = &device->msix_entries[j]; msix = &device->msix_entries[j];
chan = ioat_chan_by_index(device, j); ioat_chan = ioat_chan_by_index(device, j);
devm_free_irq(dev, msix->vector, chan); devm_free_irq(dev, msix->vector, ioat_chan);
} }
goto msi; goto msi;
} }
@ -507,14 +510,14 @@ static ssize_t
ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{ {
struct ioat_sysfs_entry *entry; struct ioat_sysfs_entry *entry;
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
entry = container_of(attr, struct ioat_sysfs_entry, attr); entry = container_of(attr, struct ioat_sysfs_entry, attr);
chan = container_of(kobj, struct ioat_chan_common, kobj); ioat_chan = container_of(kobj, struct ioatdma_chan, kobj);
if (!entry->show) if (!entry->show)
return -EIO; return -EIO;
return entry->show(&chan->common, page); return entry->show(&ioat_chan->dma_chan, page);
} }
const struct sysfs_ops ioat_sysfs_ops = { const struct sysfs_ops ioat_sysfs_ops = {
@ -527,16 +530,17 @@ void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
struct dma_chan *c; struct dma_chan *c;
list_for_each_entry(c, &dma->channels, device_node) { list_for_each_entry(c, &dma->channels, device_node) {
struct ioat_chan_common *chan = to_chan_common(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct kobject *parent = &c->dev->device.kobj; struct kobject *parent = &c->dev->device.kobj;
int err; int err;
err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata"); err = kobject_init_and_add(&ioat_chan->kobj, type,
parent, "quickdata");
if (err) { if (err) {
dev_warn(to_dev(chan), dev_warn(to_dev(ioat_chan),
"sysfs init error (%d), continuing...\n", err); "sysfs init error (%d), continuing...\n", err);
kobject_put(&chan->kobj); kobject_put(&ioat_chan->kobj);
set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state); set_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state);
} }
} }
} }
@ -547,11 +551,11 @@ void ioat_kobject_del(struct ioatdma_device *device)
struct dma_chan *c; struct dma_chan *c;
list_for_each_entry(c, &dma->channels, device_node) { list_for_each_entry(c, &dma->channels, device_node) {
struct ioat_chan_common *chan = to_chan_common(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) { if (!test_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state)) {
kobject_del(&chan->kobj); kobject_del(&ioat_chan->kobj);
kobject_put(&chan->kobj); kobject_put(&ioat_chan->kobj);
} }
} }
} }

89
drivers/dma/ioat/dma.h
View File

@ -78,20 +78,20 @@ struct ioatdma_device {
struct dma_device common; struct dma_device common;
u8 version; u8 version;
struct msix_entry msix_entries[4]; struct msix_entry msix_entries[4];
struct ioat_chan_common *idx[4]; struct ioatdma_chan *idx[4];
struct dca_provider *dca; struct dca_provider *dca;
enum ioat_irq_mode irq_mode; enum ioat_irq_mode irq_mode;
u32 cap; u32 cap;
void (*intr_quirk)(struct ioatdma_device *device); void (*intr_quirk)(struct ioatdma_device *device);
int (*enumerate_channels)(struct ioatdma_device *device); int (*enumerate_channels)(struct ioatdma_device *device);
int (*reset_hw)(struct ioat_chan_common *chan); int (*reset_hw)(struct ioatdma_chan *ioat_chan);
void (*cleanup_fn)(unsigned long data); void (*cleanup_fn)(unsigned long data);
void (*timer_fn)(unsigned long data); void (*timer_fn)(unsigned long data);
int (*self_test)(struct ioatdma_device *device); int (*self_test)(struct ioatdma_device *device);
}; };
struct ioat_chan_common { struct ioatdma_chan {
struct dma_chan common; struct dma_chan dma_chan;
void __iomem *reg_base; void __iomem *reg_base;
dma_addr_t last_completion; dma_addr_t last_completion;
spinlock_t cleanup_lock; spinlock_t cleanup_lock;
@ -112,6 +112,27 @@ struct ioat_chan_common {
u64 *completion; u64 *completion;
struct tasklet_struct cleanup_task; struct tasklet_struct cleanup_task;
struct kobject kobj; struct kobject kobj;
/* ioat v2 / v3 channel attributes
* @xfercap_log; log2 of channel max transfer length (for fast division)
* @head: allocated index
* @issued: hardware notification point
* @tail: cleanup index
* @dmacount: identical to 'head' except for occasionally resetting to zero
* @alloc_order: log2 of the number of allocated descriptors
* @produce: number of descriptors to produce at submit time
* @ring: software ring buffer implementation of hardware ring
* @prep_lock: serializes descriptor preparation (producers)
*/
size_t xfercap_log;
u16 head;
u16 issued;
u16 tail;
u16 dmacount;
u16 alloc_order;
u16 produce;
struct ioat_ring_ent **ring;
spinlock_t prep_lock;
}; };
struct ioat_sysfs_entry { struct ioat_sysfs_entry {
@ -133,11 +154,13 @@ struct ioat_sed_ent {
unsigned int hw_pool; unsigned int hw_pool;
}; };
static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
{ {
return container_of(c, struct ioat_chan_common, common); return container_of(c, struct ioatdma_chan, dma_chan);
} }
/* wrapper around hardware descriptor format + additional software fields */ /* wrapper around hardware descriptor format + additional software fields */
#ifdef DEBUG #ifdef DEBUG
@ -149,10 +172,10 @@ static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
#endif #endif
static inline void static inline void
__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw,
struct dma_async_tx_descriptor *tx, int id) struct dma_async_tx_descriptor *tx, int id)
{ {
struct device *dev = to_dev(chan); struct device *dev = to_dev(ioat_chan);
dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x" dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
" ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id, " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id,
@ -162,25 +185,25 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
} }
#define dump_desc_dbg(c, d) \ #define dump_desc_dbg(c, d) \
({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; })
static inline struct ioat_chan_common * static inline struct ioatdma_chan *
ioat_chan_by_index(struct ioatdma_device *device, int index) ioat_chan_by_index(struct ioatdma_device *device, int index)
{ {
return device->idx[index]; return device->idx[index];
} }
static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) static inline u64 ioat_chansts_32(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
u64 status; u64 status;
u32 status_lo; u32 status_lo;
/* We need to read the low address first as this causes the /* We need to read the low address first as this causes the
* chipset to latch the upper bits for the subsequent read * chipset to latch the upper bits for the subsequent read
*/ */
status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); status_lo = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); status = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
status <<= 32; status <<= 32;
status |= status_lo; status |= status_lo;
@ -189,16 +212,16 @@ static inline u64 ioat_chansts_32(struct ioat_chan_common *chan)
#if BITS_PER_LONG == 64 #if BITS_PER_LONG == 64
static inline u64 ioat_chansts(struct ioat_chan_common *chan) static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
u64 status; u64 status;
/* With IOAT v3.3 the status register is 64bit. */ /* With IOAT v3.3 the status register is 64bit. */
if (ver >= IOAT_VER_3_3) if (ver >= IOAT_VER_3_3)
status = readq(chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); status = readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET(ver));
else else
status = ioat_chansts_32(chan); status = ioat_chansts_32(ioat_chan);
return status; return status;
} }
@ -212,31 +235,33 @@ static inline u64 ioat_chansts_to_addr(u64 status)
return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
} }
static inline u32 ioat_chanerr(struct ioat_chan_common *chan) static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan)
{ {
return readl(chan->reg_base + IOAT_CHANERR_OFFSET); return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
} }
static inline void ioat_suspend(struct ioat_chan_common *chan) static inline void ioat_suspend(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); writeb(IOAT_CHANCMD_SUSPEND,
ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
} }
static inline void ioat_reset(struct ioat_chan_common *chan) static inline void ioat_reset(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); writeb(IOAT_CHANCMD_RESET,
ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
} }
static inline bool ioat_reset_pending(struct ioat_chan_common *chan) static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
u8 cmd; u8 cmd;
cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET; return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
} }
@ -272,15 +297,15 @@ int ioat_dma_self_test(struct ioatdma_device *device);
void ioat_dma_remove(struct ioatdma_device *device); void ioat_dma_remove(struct ioatdma_device *device);
struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
void ioat_init_channel(struct ioatdma_device *device, void ioat_init_channel(struct ioatdma_device *device,
struct ioat_chan_common *chan, int idx); struct ioatdma_chan *ioat_chan, int idx);
enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
struct dma_tx_state *txstate); struct dma_tx_state *txstate);
bool ioat_cleanup_preamble(struct ioat_chan_common *chan, bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
dma_addr_t *phys_complete); dma_addr_t *phys_complete);
void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
void ioat_kobject_del(struct ioatdma_device *device); void ioat_kobject_del(struct ioatdma_device *device);
int ioat_dma_setup_interrupts(struct ioatdma_device *device); int ioat_dma_setup_interrupts(struct ioatdma_device *device);
void ioat_stop(struct ioat_chan_common *chan); void ioat_stop(struct ioatdma_chan *ioat_chan);
extern const struct sysfs_ops ioat_sysfs_ops; extern const struct sysfs_ops ioat_sysfs_ops;
extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_version_attr;
extern struct ioat_sysfs_entry ioat_cap_attr; extern struct ioat_sysfs_entry ioat_cap_attr;

349
drivers/dma/ioat/dma_v2.c
View File

@ -48,26 +48,26 @@ module_param(ioat_ring_max_alloc_order, int, 0644);
MODULE_PARM_DESC(ioat_ring_max_alloc_order, MODULE_PARM_DESC(ioat_ring_max_alloc_order,
"ioat2+: upper limit for ring size (default: 16)"); "ioat2+: upper limit for ring size (default: 16)");
void __ioat2_issue_pending(struct ioat2_dma_chan *ioat) void __ioat2_issue_pending(struct ioatdma_chan *ioat_chan)
{ {
struct ioat_chan_common *chan = &ioat->base; ioat_chan->dmacount += ioat2_ring_pending(ioat_chan);
ioat_chan->issued = ioat_chan->head;
ioat->dmacount += ioat2_ring_pending(ioat); writew(ioat_chan->dmacount,
ioat->issued = ioat->head; ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); dev_dbg(to_dev(ioat_chan),
dev_dbg(to_dev(chan),
"%s: head: %#x tail: %#x issued: %#x count: %#x\n", "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); __func__, ioat_chan->head, ioat_chan->tail,
ioat_chan->issued, ioat_chan->dmacount);
} }
void ioat2_issue_pending(struct dma_chan *c) void ioat2_issue_pending(struct dma_chan *c)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
if (ioat2_ring_pending(ioat)) { if (ioat2_ring_pending(ioat_chan)) {
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
__ioat2_issue_pending(ioat); __ioat2_issue_pending(ioat_chan);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
} }
} }
@ -78,26 +78,27 @@ void ioat2_issue_pending(struct dma_chan *c)
* Check if the number of unsubmitted descriptors has exceeded the * Check if the number of unsubmitted descriptors has exceeded the
* watermark. Called with prep_lock held * watermark. Called with prep_lock held
*/ */
static void ioat2_update_pending(struct ioat2_dma_chan *ioat) static void ioat2_update_pending(struct ioatdma_chan *ioat_chan)
{ {
if (ioat2_ring_pending(ioat) > ioat_pending_level) if (ioat2_ring_pending(ioat_chan) > ioat_pending_level)
__ioat2_issue_pending(ioat); __ioat2_issue_pending(ioat_chan);
} }
static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) static void __ioat2_start_null_desc(struct ioatdma_chan *ioat_chan)
{ {
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
struct ioat_dma_descriptor *hw; struct ioat_dma_descriptor *hw;
if (ioat2_ring_space(ioat) < 1) { if (ioat2_ring_space(ioat_chan) < 1) {
dev_err(to_dev(&ioat->base), dev_err(to_dev(ioat_chan),
"Unable to start null desc - ring full\n"); "Unable to start null desc - ring full\n");
return; return;
} }
dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n", dev_dbg(to_dev(ioat_chan),
__func__, ioat->head, ioat->tail, ioat->issued); "%s: head: %#x tail: %#x issued: %#x\n",
desc = ioat2_get_ring_ent(ioat, ioat->head); __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head);
hw = desc->hw; hw = desc->hw;
hw->ctl = 0; hw->ctl = 0;
@ -109,72 +110,71 @@ static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
hw->src_addr = 0; hw->src_addr = 0;
hw->dst_addr = 0; hw->dst_addr = 0;
async_tx_ack(&desc->txd); async_tx_ack(&desc->txd);
ioat2_set_chainaddr(ioat, desc->txd.phys); ioat2_set_chainaddr(ioat_chan, desc->txd.phys);
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
wmb(); wmb();
ioat->head += 1; ioat_chan->head += 1;
__ioat2_issue_pending(ioat); __ioat2_issue_pending(ioat_chan);
} }
static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat) static void ioat2_start_null_desc(struct ioatdma_chan *ioat_chan)
{ {
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
__ioat2_start_null_desc(ioat); __ioat2_start_null_desc(ioat_chan);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
} }
void __ioat2_restart_chan(struct ioat2_dma_chan *ioat) void __ioat2_restart_chan(struct ioatdma_chan *ioat_chan)
{ {
struct ioat_chan_common *chan = &ioat->base;
/* set the tail to be re-issued */ /* set the tail to be re-issued */
ioat->issued = ioat->tail; ioat_chan->issued = ioat_chan->tail;
ioat->dmacount = 0; ioat_chan->dmacount = 0;
set_bit(IOAT_COMPLETION_PENDING, &chan->state); set_bit(IOAT_COMPLETION_PENDING, &ioat_chan->state);
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
dev_dbg(to_dev(chan), dev_dbg(to_dev(ioat_chan),
"%s: head: %#x tail: %#x issued: %#x count: %#x\n", "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); __func__, ioat_chan->head, ioat_chan->tail,
ioat_chan->issued, ioat_chan->dmacount);
if (ioat2_ring_pending(ioat)) { if (ioat2_ring_pending(ioat_chan)) {
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
desc = ioat2_get_ring_ent(ioat, ioat->tail); desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail);
ioat2_set_chainaddr(ioat, desc->txd.phys); ioat2_set_chainaddr(ioat_chan, desc->txd.phys);
__ioat2_issue_pending(ioat); __ioat2_issue_pending(ioat_chan);
} else } else
__ioat2_start_null_desc(ioat); __ioat2_start_null_desc(ioat_chan);
} }
int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo) int ioat2_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
{ {
unsigned long end = jiffies + tmo; unsigned long end = jiffies + tmo;
int err = 0; int err = 0;
u32 status; u32 status;
status = ioat_chansts(chan); status = ioat_chansts(ioat_chan);
if (is_ioat_active(status) || is_ioat_idle(status)) if (is_ioat_active(status) || is_ioat_idle(status))
ioat_suspend(chan); ioat_suspend(ioat_chan);
while (is_ioat_active(status) || is_ioat_idle(status)) { while (is_ioat_active(status) || is_ioat_idle(status)) {
if (tmo && time_after(jiffies, end)) { if (tmo && time_after(jiffies, end)) {
err = -ETIMEDOUT; err = -ETIMEDOUT;
break; break;
} }
status = ioat_chansts(chan); status = ioat_chansts(ioat_chan);
cpu_relax(); cpu_relax();
} }
return err; return err;
} }
int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo) int ioat2_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
{ {
unsigned long end = jiffies + tmo; unsigned long end = jiffies + tmo;
int err = 0; int err = 0;
ioat_reset(chan); ioat_reset(ioat_chan);
while (ioat_reset_pending(chan)) { while (ioat_reset_pending(ioat_chan)) {
if (end && time_after(jiffies, end)) { if (end && time_after(jiffies, end)) {
err = -ETIMEDOUT; err = -ETIMEDOUT;
break; break;
@ -191,7 +191,7 @@ int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo)
*/ */
int ioat2_enumerate_channels(struct ioatdma_device *device) int ioat2_enumerate_channels(struct ioatdma_device *device)
{ {
struct ioat2_dma_chan *ioat; struct ioatdma_chan *ioat_chan;
struct device *dev = &device->pdev->dev; struct device *dev = &device->pdev->dev;
struct dma_device *dma = &device->common; struct dma_device *dma = &device->common;
u8 xfercap_log; u8 xfercap_log;
@ -212,14 +212,14 @@ int ioat2_enumerate_channels(struct ioatdma_device *device)
dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
for (i = 0; i < dma->chancnt; i++) { for (i = 0; i < dma->chancnt; i++) {
ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
if (!ioat) if (!ioat_chan)
break; break;
ioat_init_channel(device, &ioat->base, i); ioat_init_channel(device, ioat_chan, i);
ioat->xfercap_log = xfercap_log; ioat_chan->xfercap_log = xfercap_log;
spin_lock_init(&ioat->prep_lock); spin_lock_init(&ioat_chan->prep_lock);
if (device->reset_hw(&ioat->base)) { if (device->reset_hw(ioat_chan)) {
i = 0; i = 0;
break; break;
} }
@ -231,15 +231,14 @@ int ioat2_enumerate_channels(struct ioatdma_device *device)
static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx) static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
{ {
struct dma_chan *c = tx->chan; struct dma_chan *c = tx->chan;
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_chan_common *chan = &ioat->base;
dma_cookie_t cookie; dma_cookie_t cookie;
cookie = dma_cookie_assign(tx); cookie = dma_cookie_assign(tx);
dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &chan->state)) if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
/* make descriptor updates visible before advancing ioat->head, /* make descriptor updates visible before advancing ioat->head,
* this is purposefully not smp_wmb() since we are also * this is purposefully not smp_wmb() since we are also
@ -247,10 +246,10 @@ static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
*/ */
wmb(); wmb();
ioat->head += ioat->produce; ioat_chan->head += ioat_chan->produce;
ioat2_update_pending(ioat); ioat2_update_pending(ioat_chan);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
return cookie; return cookie;
} }
@ -333,79 +332,78 @@ void ioat2_free_chan_resources(struct dma_chan *c);
*/ */
int ioat2_alloc_chan_resources(struct dma_chan *c) int ioat2_alloc_chan_resources(struct dma_chan *c)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_chan_common *chan = &ioat->base;
struct ioat_ring_ent **ring; struct ioat_ring_ent **ring;
u64 status; u64 status;
int order; int order;
int i = 0; int i = 0;
/* have we already been set up? */ /* have we already been set up? */
if (ioat->ring) if (ioat_chan->ring)
return 1 << ioat->alloc_order; return 1 << ioat_chan->alloc_order;
/* Setup register to interrupt and write completion status on error */ /* Setup register to interrupt and write completion status on error */
writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
/* allocate a completion writeback area */ /* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */ /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
chan->completion = pci_pool_alloc(chan->device->completion_pool, ioat_chan->completion =
GFP_KERNEL, &chan->completion_dma); pci_pool_alloc(ioat_chan->device->completion_pool,
if (!chan->completion) GFP_KERNEL, &ioat_chan->completion_dma);
if (!ioat_chan->completion)
return -ENOMEM; return -ENOMEM;
memset(chan->completion, 0, sizeof(*chan->completion)); memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion));
writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF, writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,
chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
writel(((u64) chan->completion_dma) >> 32, writel(((u64)ioat_chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
order = ioat_get_alloc_order(); order = ioat_get_alloc_order();
ring = ioat2_alloc_ring(c, order, GFP_KERNEL); ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
if (!ring) if (!ring)
return -ENOMEM; return -ENOMEM;
spin_lock_bh(&chan->cleanup_lock); spin_lock_bh(&ioat_chan->cleanup_lock);
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
ioat->ring = ring; ioat_chan->ring = ring;
ioat->head = 0; ioat_chan->head = 0;
ioat->issued = 0; ioat_chan->issued = 0;
ioat->tail = 0; ioat_chan->tail = 0;
ioat->alloc_order = order; ioat_chan->alloc_order = order;
set_bit(IOAT_RUN, &chan->state); set_bit(IOAT_RUN, &ioat_chan->state);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
spin_unlock_bh(&chan->cleanup_lock); spin_unlock_bh(&ioat_chan->cleanup_lock);
ioat2_start_null_desc(ioat); ioat2_start_null_desc(ioat_chan);
/* check that we got off the ground */ /* check that we got off the ground */
do { do {
udelay(1); udelay(1);
status = ioat_chansts(chan); status = ioat_chansts(ioat_chan);
} while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status)); } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
if (is_ioat_active(status) || is_ioat_idle(status)) { if (is_ioat_active(status) || is_ioat_idle(status)) {
return 1 << ioat->alloc_order; return 1 << ioat_chan->alloc_order;
} else { } else {
u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
dev_WARN(to_dev(chan), dev_WARN(to_dev(ioat_chan),
"failed to start channel chanerr: %#x\n", chanerr); "failed to start channel chanerr: %#x\n", chanerr);
ioat2_free_chan_resources(c); ioat2_free_chan_resources(c);
return -EFAULT; return -EFAULT;
} }
} }
bool reshape_ring(struct ioat2_dma_chan *ioat, int order) bool reshape_ring(struct ioatdma_chan *ioat_chan, int order)
{ {
/* reshape differs from normal ring allocation in that we want /* reshape differs from normal ring allocation in that we want
* to allocate a new software ring while only * to allocate a new software ring while only
* extending/truncating the hardware ring * extending/truncating the hardware ring
*/ */
struct ioat_chan_common *chan = &ioat->base; struct dma_chan *c = &ioat_chan->dma_chan;
struct dma_chan *c = &chan->common; const u32 curr_size = ioat2_ring_size(ioat_chan);
const u32 curr_size = ioat2_ring_size(ioat); const u16 active = ioat2_ring_active(ioat_chan);
const u16 active = ioat2_ring_active(ioat);
const u32 new_size = 1 << order; const u32 new_size = 1 << order;
struct ioat_ring_ent **ring; struct ioat_ring_ent **ring;
u32 i; u32 i;
@ -432,21 +430,22 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
if (new_size > curr_size) { if (new_size > curr_size) {
/* copy current descriptors to the new ring */ /* copy current descriptors to the new ring */
for (i = 0; i < curr_size; i++) { for (i = 0; i < curr_size; i++) {
u16 curr_idx = (ioat->tail+i) & (curr_size-1); u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
u16 new_idx = (ioat->tail+i) & (new_size-1); u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
ring[new_idx] = ioat->ring[curr_idx]; ring[new_idx] = ioat_chan->ring[curr_idx];
set_desc_id(ring[new_idx], new_idx); set_desc_id(ring[new_idx], new_idx);
} }
/* add new descriptors to the ring */ /* add new descriptors to the ring */
for (i = curr_size; i < new_size; i++) { for (i = curr_size; i < new_size; i++) {
u16 new_idx = (ioat->tail+i) & (new_size-1); u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT); ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
if (!ring[new_idx]) { if (!ring[new_idx]) {
while (i--) { while (i--) {
u16 new_idx = (ioat->tail+i) & (new_size-1); u16 new_idx = (ioat_chan->tail+i) &
(new_size-1);
ioat2_free_ring_ent(ring[new_idx], c); ioat2_free_ring_ent(ring[new_idx], c);
} }
@ -458,7 +457,7 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
/* hw link new descriptors */ /* hw link new descriptors */
for (i = curr_size-1; i < new_size; i++) { for (i = curr_size-1; i < new_size; i++) {
u16 new_idx = (ioat->tail+i) & (new_size-1); u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)]; struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
struct ioat_dma_descriptor *hw = ring[new_idx]->hw; struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
@ -472,10 +471,10 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
* removed descriptors * removed descriptors
*/ */
for (i = 0; i < new_size; i++) { for (i = 0; i < new_size; i++) {
u16 curr_idx = (ioat->tail+i) & (curr_size-1); u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
u16 new_idx = (ioat->tail+i) & (new_size-1); u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
ring[new_idx] = ioat->ring[curr_idx]; ring[new_idx] = ioat_chan->ring[curr_idx];
set_desc_id(ring[new_idx], new_idx); set_desc_id(ring[new_idx], new_idx);
} }
@ -483,22 +482,22 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
for (i = new_size; i < curr_size; i++) { for (i = new_size; i < curr_size; i++) {
struct ioat_ring_ent *ent; struct ioat_ring_ent *ent;
ent = ioat2_get_ring_ent(ioat, ioat->tail+i); ent = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail+i);
ioat2_free_ring_ent(ent, c); ioat2_free_ring_ent(ent, c);
} }
/* fix up hardware ring */ /* fix up hardware ring */
hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw; hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw;
next = ring[(ioat->tail+new_size) & (new_size-1)]; next = ring[(ioat_chan->tail+new_size) & (new_size-1)];
hw->next = next->txd.phys; hw->next = next->txd.phys;
} }
dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n",
__func__, new_size); __func__, new_size);
kfree(ioat->ring); kfree(ioat_chan->ring);
ioat->ring = ring; ioat_chan->ring = ring;
ioat->alloc_order = order; ioat_chan->alloc_order = order;
return true; return true;
} }
@ -508,55 +507,57 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
* @ioat: ioat2,3 channel (ring) to operate on * @ioat: ioat2,3 channel (ring) to operate on
* @num_descs: allocation length * @num_descs: allocation length
*/ */
int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs) int ioat2_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
{ {
struct ioat_chan_common *chan = &ioat->base;
bool retry; bool retry;
retry: retry:
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
/* never allow the last descriptor to be consumed, we need at /* never allow the last descriptor to be consumed, we need at
* least one free at all times to allow for on-the-fly ring * least one free at all times to allow for on-the-fly ring
* resizing. * resizing.
*/ */
if (likely(ioat2_ring_space(ioat) > num_descs)) { if (likely(ioat2_ring_space(ioat_chan) > num_descs)) {
dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n", dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
__func__, num_descs, ioat->head, ioat->tail, ioat->issued); __func__, num_descs, ioat_chan->head,
ioat->produce = num_descs; ioat_chan->tail, ioat_chan->issued);
ioat_chan->produce = num_descs;
return 0; /* with ioat->prep_lock held */ return 0; /* with ioat->prep_lock held */
} }
retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &chan->state); retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
/* is another cpu already trying to expand the ring? */ /* is another cpu already trying to expand the ring? */
if (retry) if (retry)
goto retry; goto retry;
spin_lock_bh(&chan->cleanup_lock); spin_lock_bh(&ioat_chan->cleanup_lock);
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
retry = reshape_ring(ioat, ioat->alloc_order + 1); retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1);
clear_bit(IOAT_RESHAPE_PENDING, &chan->state); clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
spin_unlock_bh(&chan->cleanup_lock); spin_unlock_bh(&ioat_chan->cleanup_lock);
/* if we were able to expand the ring retry the allocation */ /* if we were able to expand the ring retry the allocation */
if (retry) if (retry)
goto retry; goto retry;
if (printk_ratelimit()) if (printk_ratelimit())
dev_dbg(to_dev(chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n", dev_dbg(to_dev(ioat_chan),
__func__, num_descs, ioat->head, ioat->tail, ioat->issued); "%s: ring full! num_descs: %d (%x:%x:%x)\n",
__func__, num_descs, ioat_chan->head,
ioat_chan->tail, ioat_chan->issued);
/* progress reclaim in the allocation failure case we may be /* progress reclaim in the allocation failure case we may be
* called under bh_disabled so we need to trigger the timer * called under bh_disabled so we need to trigger the timer
* event directly * event directly
*/ */
if (time_is_before_jiffies(chan->timer.expires) if (time_is_before_jiffies(ioat_chan->timer.expires)
&& timer_pending(&chan->timer)) { && timer_pending(&ioat_chan->timer)) {
struct ioatdma_device *device = chan->device; struct ioatdma_device *device = ioat_chan->device;
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
device->timer_fn((unsigned long) &chan->common); device->timer_fn((unsigned long)ioat_chan);
} }
return -ENOMEM; return -ENOMEM;
@ -566,7 +567,7 @@ struct dma_async_tx_descriptor *
ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
dma_addr_t dma_src, size_t len, unsigned long flags) dma_addr_t dma_src, size_t len, unsigned long flags)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_dma_descriptor *hw; struct ioat_dma_descriptor *hw;
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
dma_addr_t dst = dma_dest; dma_addr_t dst = dma_dest;
@ -574,16 +575,17 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
size_t total_len = len; size_t total_len = len;
int num_descs, idx, i; int num_descs, idx, i;
num_descs = ioat2_xferlen_to_descs(ioat, len); num_descs = ioat2_xferlen_to_descs(ioat_chan, len);
if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0) if (likely(num_descs) &&
idx = ioat->head; ioat2_check_space_lock(ioat_chan, num_descs) == 0)
idx = ioat_chan->head;
else else
return NULL; return NULL;
i = 0; i = 0;
do { do {
size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log); size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);
desc = ioat2_get_ring_ent(ioat, idx + i); desc = ioat2_get_ring_ent(ioat_chan, idx + i);
hw = desc->hw; hw = desc->hw;
hw->size = copy; hw->size = copy;
@ -594,7 +596,7 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
len -= copy; len -= copy;
dst += copy; dst += copy;
src += copy; src += copy;
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
} while (++i < num_descs); } while (++i < num_descs);
desc->txd.flags = flags; desc->txd.flags = flags;
@ -602,7 +604,7 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
hw->ctl_f.compl_write = 1; hw->ctl_f.compl_write = 1;
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
/* we leave the channel locked to ensure in order submission */ /* we leave the channel locked to ensure in order submission */
return &desc->txd; return &desc->txd;
@ -614,69 +616,68 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
*/ */
void ioat2_free_chan_resources(struct dma_chan *c) void ioat2_free_chan_resources(struct dma_chan *c)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_chan_common *chan = &ioat->base; struct ioatdma_device *device = ioat_chan->device;
struct ioatdma_device *device = chan->device;
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
const int total_descs = 1 << ioat->alloc_order; const int total_descs = 1 << ioat_chan->alloc_order;
int descs; int descs;
int i; int i;
/* Before freeing channel resources first check /* Before freeing channel resources first check
* if they have been previously allocated for this channel. * if they have been previously allocated for this channel.
*/ */
if (!ioat->ring) if (!ioat_chan->ring)
return; return;
ioat_stop(chan); ioat_stop(ioat_chan);
device->reset_hw(chan); device->reset_hw(ioat_chan);
spin_lock_bh(&chan->cleanup_lock); spin_lock_bh(&ioat_chan->cleanup_lock);
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
descs = ioat2_ring_space(ioat); descs = ioat2_ring_space(ioat_chan);
dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs); dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
for (i = 0; i < descs; i++) { for (i = 0; i < descs; i++) {
desc = ioat2_get_ring_ent(ioat, ioat->head + i); desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head + i);
ioat2_free_ring_ent(desc, c); ioat2_free_ring_ent(desc, c);
} }
if (descs < total_descs) if (descs < total_descs)
dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
total_descs - descs); total_descs - descs);
for (i = 0; i < total_descs - descs; i++) { for (i = 0; i < total_descs - descs; i++) {
desc = ioat2_get_ring_ent(ioat, ioat->tail + i); desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail + i);
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
ioat2_free_ring_ent(desc, c); ioat2_free_ring_ent(desc, c);
} }
kfree(ioat->ring); kfree(ioat_chan->ring);
ioat->ring = NULL; ioat_chan->ring = NULL;
ioat->alloc_order = 0; ioat_chan->alloc_order = 0;
pci_pool_free(device->completion_pool, chan->completion, pci_pool_free(device->completion_pool, ioat_chan->completion,
chan->completion_dma); ioat_chan->completion_dma);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
spin_unlock_bh(&chan->cleanup_lock); spin_unlock_bh(&ioat_chan->cleanup_lock);
chan->last_completion = 0; ioat_chan->last_completion = 0;
chan->completion_dma = 0; ioat_chan->completion_dma = 0;
ioat->dmacount = 0; ioat_chan->dmacount = 0;
} }
static ssize_t ring_size_show(struct dma_chan *c, char *page) static ssize_t ring_size_show(struct dma_chan *c, char *page)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1); return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1);
} }
static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
static ssize_t ring_active_show(struct dma_chan *c, char *page) static ssize_t ring_active_show(struct dma_chan *c, char *page)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
/* ...taken outside the lock, no need to be precise */ /* ...taken outside the lock, no need to be precise */
return sprintf(page, "%d\n", ioat2_ring_active(ioat)); return sprintf(page, "%d\n", ioat2_ring_active(ioat_chan));
} }
static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);

82
drivers/dma/ioat/dma_v2.h
View File

@ -38,65 +38,36 @@ extern int ioat_ring_alloc_order;
#define ioat_get_max_alloc_order() \ #define ioat_get_max_alloc_order() \
(min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER)) (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes static inline u32 ioat2_ring_size(struct ioatdma_chan *ioat_chan)
* @base: common ioat channel parameters
* @xfercap_log; log2 of channel max transfer length (for fast division)
* @head: allocated index
* @issued: hardware notification point
* @tail: cleanup index
* @dmacount: identical to 'head' except for occasionally resetting to zero
* @alloc_order: log2 of the number of allocated descriptors
* @produce: number of descriptors to produce at submit time
* @ring: software ring buffer implementation of hardware ring
* @prep_lock: serializes descriptor preparation (producers)
*/
struct ioat2_dma_chan {
struct ioat_chan_common base;
size_t xfercap_log;
u16 head;
u16 issued;
u16 tail;
u16 dmacount;
u16 alloc_order;
u16 produce;
struct ioat_ring_ent **ring;
spinlock_t prep_lock;
};
static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
{ {
struct ioat_chan_common *chan = to_chan_common(c); return 1 << ioat_chan->alloc_order;
return container_of(chan, struct ioat2_dma_chan, base);
}
static inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat)
{
return 1 << ioat->alloc_order;
} }
/* count of descriptors in flight with the engine */ /* count of descriptors in flight with the engine */
static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat) static inline u16 ioat2_ring_active(struct ioatdma_chan *ioat_chan)
{ {
return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat)); return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
ioat2_ring_size(ioat_chan));
} }
/* count of descriptors pending submission to hardware */ /* count of descriptors pending submission to hardware */
static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat) static inline u16 ioat2_ring_pending(struct ioatdma_chan *ioat_chan)
{ {
return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat)); return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
ioat2_ring_size(ioat_chan));
} }
static inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat) static inline u32 ioat2_ring_space(struct ioatdma_chan *ioat_chan)
{ {
return ioat2_ring_size(ioat) - ioat2_ring_active(ioat); return ioat2_ring_size(ioat_chan) - ioat2_ring_active(ioat_chan);
} }
static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len) static inline u16
ioat2_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
{ {
u16 num_descs = len >> ioat->xfercap_log; u16 num_descs = len >> ioat_chan->xfercap_log;
num_descs += !!(len & ((1 << ioat->xfercap_log) - 1)); num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
return num_descs; return num_descs;
} }
@ -136,25 +107,24 @@ struct ioat_ring_ent {
}; };
static inline struct ioat_ring_ent * static inline struct ioat_ring_ent *
ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx) ioat2_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
{ {
return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)]; return ioat_chan->ring[idx & (ioat2_ring_size(ioat_chan) - 1)];
} }
static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr) static inline void
ioat2_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
{ {
struct ioat_chan_common *chan = &ioat->base;
writel(addr & 0x00000000FFFFFFFF, writel(addr & 0x00000000FFFFFFFF,
chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
writel(addr >> 32, writel(addr >> 32,
chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
} }
int ioat2_dma_probe(struct ioatdma_device *dev, int dca); int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
int ioat3_dma_probe(struct ioatdma_device *dev, int dca); int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); int ioat2_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
int ioat2_enumerate_channels(struct ioatdma_device *device); int ioat2_enumerate_channels(struct ioatdma_device *device);
struct dma_async_tx_descriptor * struct dma_async_tx_descriptor *
ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
@ -162,12 +132,12 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
void ioat2_issue_pending(struct dma_chan *chan); void ioat2_issue_pending(struct dma_chan *chan);
int ioat2_alloc_chan_resources(struct dma_chan *c); int ioat2_alloc_chan_resources(struct dma_chan *c);
void ioat2_free_chan_resources(struct dma_chan *c); void ioat2_free_chan_resources(struct dma_chan *c);
void __ioat2_restart_chan(struct ioat2_dma_chan *ioat); void __ioat2_restart_chan(struct ioatdma_chan *ioat_chan);
bool reshape_ring(struct ioat2_dma_chan *ioat, int order); bool reshape_ring(struct ioatdma_chan *ioat, int order);
void __ioat2_issue_pending(struct ioat2_dma_chan *ioat); void __ioat2_issue_pending(struct ioatdma_chan *ioat_chan);
void ioat2_timer_event(unsigned long data); void ioat2_timer_event(unsigned long data);
int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); int ioat2_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo);
int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); int ioat2_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo);
extern struct kobj_type ioat2_ktype; extern struct kobj_type ioat2_ktype;
extern struct kmem_cache *ioat2_cache; extern struct kmem_cache *ioat2_cache;
#endif /* IOATDMA_V2_H */ #endif /* IOATDMA_V2_H */

307
drivers/dma/ioat/dma_v3.c
View File

@ -85,7 +85,7 @@ static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 4, 5, 6 }; 0, 1, 2, 3, 4, 5, 6 };
static void ioat3_eh(struct ioat2_dma_chan *ioat); static void ioat3_eh(struct ioatdma_chan *ioat_chan);
static void xor_set_src(struct ioat_raw_descriptor *descs[2], static void xor_set_src(struct ioat_raw_descriptor *descs[2],
dma_addr_t addr, u32 offset, int idx) dma_addr_t addr, u32 offset, int idx)
@ -304,35 +304,35 @@ static bool desc_has_ext(struct ioat_ring_ent *desc)
return false; return false;
} }
static u64 ioat3_get_current_completion(struct ioat_chan_common *chan) static u64 ioat3_get_current_completion(struct ioatdma_chan *ioat_chan)
{ {
u64 phys_complete; u64 phys_complete;
u64 completion; u64 completion;
completion = *chan->completion; completion = *ioat_chan->completion;
phys_complete = ioat_chansts_to_addr(completion); phys_complete = ioat_chansts_to_addr(completion);
dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
(unsigned long long) phys_complete); (unsigned long long) phys_complete);
return phys_complete; return phys_complete;
} }
static bool ioat3_cleanup_preamble(struct ioat_chan_common *chan, static bool ioat3_cleanup_preamble(struct ioatdma_chan *ioat_chan,
u64 *phys_complete) u64 *phys_complete)
{ {
*phys_complete = ioat3_get_current_completion(chan); *phys_complete = ioat3_get_current_completion(ioat_chan);
if (*phys_complete == chan->last_completion) if (*phys_complete == ioat_chan->last_completion)
return false; return false;
clear_bit(IOAT_COMPLETION_ACK, &chan->state); clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
return true; return true;
} }
static void static void
desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc) desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
{ {
struct ioat_dma_descriptor *hw = desc->hw; struct ioat_dma_descriptor *hw = desc->hw;
@ -368,17 +368,16 @@ desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc)
* The difference from the dma_v2.c __cleanup() is that this routine * The difference from the dma_v2.c __cleanup() is that this routine
* handles extended descriptors and dma-unmapping raid operations. * handles extended descriptors and dma-unmapping raid operations.
*/ */
static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
{ {
struct ioat_chan_common *chan = &ioat->base; struct ioatdma_device *device = ioat_chan->device;
struct ioatdma_device *device = chan->device;
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
bool seen_current = false; bool seen_current = false;
int idx = ioat->tail, i; int idx = ioat_chan->tail, i;
u16 active; u16 active;
dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued); __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
/* /*
* At restart of the channel, the completion address and the * At restart of the channel, the completion address and the
@ -390,18 +389,18 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
if (!phys_complete) if (!phys_complete)
return; return;
active = ioat2_ring_active(ioat); active = ioat2_ring_active(ioat_chan);
for (i = 0; i < active && !seen_current; i++) { for (i = 0; i < active && !seen_current; i++) {
struct dma_async_tx_descriptor *tx; struct dma_async_tx_descriptor *tx;
smp_read_barrier_depends(); smp_read_barrier_depends();
prefetch(ioat2_get_ring_ent(ioat, idx + i + 1)); prefetch(ioat2_get_ring_ent(ioat_chan, idx + i + 1));
desc = ioat2_get_ring_ent(ioat, idx + i); desc = ioat2_get_ring_ent(ioat_chan, idx + i);
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
/* set err stat if we are using dwbes */ /* set err stat if we are using dwbes */
if (device->cap & IOAT_CAP_DWBES) if (device->cap & IOAT_CAP_DWBES)
desc_get_errstat(ioat, desc); desc_get_errstat(ioat_chan, desc);
tx = &desc->txd; tx = &desc->txd;
if (tx->cookie) { if (tx->cookie) {
@ -429,70 +428,66 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
} }
} }
smp_mb(); /* finish all descriptor reads before incrementing tail */ smp_mb(); /* finish all descriptor reads before incrementing tail */
ioat->tail = idx + i; ioat_chan->tail = idx + i;
BUG_ON(active && !seen_current); /* no active descs have written a completion? */ BUG_ON(active && !seen_current); /* no active descs have written a completion? */
chan->last_completion = phys_complete; ioat_chan->last_completion = phys_complete;
if (active - i == 0) { if (active - i == 0) {
dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
__func__); __func__);
clear_bit(IOAT_COMPLETION_PENDING, &chan->state); clear_bit(IOAT_COMPLETION_PENDING, &ioat_chan->state);
mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
} }
/* 5 microsecond delay per pending descriptor */ /* 5 microsecond delay per pending descriptor */
writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK), writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
chan->device->reg_base + IOAT_INTRDELAY_OFFSET); ioat_chan->device->reg_base + IOAT_INTRDELAY_OFFSET);
} }
static void ioat3_cleanup(struct ioat2_dma_chan *ioat) static void ioat3_cleanup(struct ioatdma_chan *ioat_chan)
{ {
struct ioat_chan_common *chan = &ioat->base;
u64 phys_complete; u64 phys_complete;
spin_lock_bh(&chan->cleanup_lock); spin_lock_bh(&ioat_chan->cleanup_lock);
if (ioat3_cleanup_preamble(chan, &phys_complete)) if (ioat3_cleanup_preamble(ioat_chan, &phys_complete))
__cleanup(ioat, phys_complete); __cleanup(ioat_chan, phys_complete);
if (is_ioat_halted(*chan->completion)) { if (is_ioat_halted(*ioat_chan->completion)) {
u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
if (chanerr & IOAT_CHANERR_HANDLE_MASK) { if (chanerr & IOAT_CHANERR_HANDLE_MASK) {
mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
ioat3_eh(ioat); ioat3_eh(ioat_chan);
} }
} }
spin_unlock_bh(&chan->cleanup_lock); spin_unlock_bh(&ioat_chan->cleanup_lock);
} }
static void ioat3_cleanup_event(unsigned long data) static void ioat3_cleanup_event(unsigned long data)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
struct ioat_chan_common *chan = &ioat->base;
ioat3_cleanup(ioat); ioat3_cleanup(ioat_chan);
if (!test_bit(IOAT_RUN, &chan->state)) if (!test_bit(IOAT_RUN, &ioat_chan->state))
return; return;
writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
} }
static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) static void ioat3_restart_channel(struct ioatdma_chan *ioat_chan)
{ {
struct ioat_chan_common *chan = &ioat->base;
u64 phys_complete; u64 phys_complete;
ioat2_quiesce(chan, 0); ioat2_quiesce(ioat_chan, 0);
if (ioat3_cleanup_preamble(chan, &phys_complete)) if (ioat3_cleanup_preamble(ioat_chan, &phys_complete))
__cleanup(ioat, phys_complete); __cleanup(ioat_chan, phys_complete);
__ioat2_restart_chan(ioat); __ioat2_restart_chan(ioat_chan);
} }
static void ioat3_eh(struct ioat2_dma_chan *ioat) static void ioat3_eh(struct ioatdma_chan *ioat_chan)
{ {
struct ioat_chan_common *chan = &ioat->base; struct pci_dev *pdev = to_pdev(ioat_chan);
struct pci_dev *pdev = to_pdev(chan);
struct ioat_dma_descriptor *hw; struct ioat_dma_descriptor *hw;
struct dma_async_tx_descriptor *tx; struct dma_async_tx_descriptor *tx;
u64 phys_complete; u64 phys_complete;
@ -502,18 +497,18 @@ static void ioat3_eh(struct ioat2_dma_chan *ioat)
u32 chanerr; u32 chanerr;
/* cleanup so tail points to descriptor that caused the error */ /* cleanup so tail points to descriptor that caused the error */
if (ioat3_cleanup_preamble(chan, &phys_complete)) if (ioat3_cleanup_preamble(ioat_chan, &phys_complete))
__cleanup(ioat, phys_complete); __cleanup(ioat_chan, phys_complete);
chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int); pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
dev_dbg(to_dev(chan), "%s: error = %x:%x\n", dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
__func__, chanerr, chanerr_int); __func__, chanerr, chanerr_int);
desc = ioat2_get_ring_ent(ioat, ioat->tail); desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail);
hw = desc->hw; hw = desc->hw;
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
switch (hw->ctl_f.op) { switch (hw->ctl_f.op) {
case IOAT_OP_XOR_VAL: case IOAT_OP_XOR_VAL:
@ -537,7 +532,7 @@ static void ioat3_eh(struct ioat2_dma_chan *ioat)
/* fault on unhandled error or spurious halt */ /* fault on unhandled error or spurious halt */
if (chanerr ^ err_handled || chanerr == 0) { if (chanerr ^ err_handled || chanerr == 0) {
dev_err(to_dev(chan), "%s: fatal error (%x:%x)\n", dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
__func__, chanerr, err_handled); __func__, chanerr, err_handled);
BUG(); BUG();
} else { /* cleanup the faulty descriptor */ } else { /* cleanup the faulty descriptor */
@ -552,51 +547,48 @@ static void ioat3_eh(struct ioat2_dma_chan *ioat)
} }
} }
writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int); pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
/* mark faulting descriptor as complete */ /* mark faulting descriptor as complete */
*chan->completion = desc->txd.phys; *ioat_chan->completion = desc->txd.phys;
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
ioat3_restart_channel(ioat); ioat3_restart_channel(ioat_chan);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
} }
static void check_active(struct ioat2_dma_chan *ioat) static void check_active(struct ioatdma_chan *ioat_chan)
{ {
struct ioat_chan_common *chan = &ioat->base; if (ioat2_ring_active(ioat_chan)) {
mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
if (ioat2_ring_active(ioat)) {
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
return; return;
} }
if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state)) if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
else if (ioat->alloc_order > ioat_get_alloc_order()) { else if (ioat_chan->alloc_order > ioat_get_alloc_order()) {
/* if the ring is idle, empty, and oversized try to step /* if the ring is idle, empty, and oversized try to step
* down the size * down the size
*/ */
reshape_ring(ioat, ioat->alloc_order - 1); reshape_ring(ioat_chan, ioat_chan->alloc_order - 1);
/* keep shrinking until we get back to our minimum /* keep shrinking until we get back to our minimum
* default size * default size
*/ */
if (ioat->alloc_order > ioat_get_alloc_order()) if (ioat_chan->alloc_order > ioat_get_alloc_order())
mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
} }
} }
static void ioat3_timer_event(unsigned long data) static void ioat3_timer_event(unsigned long data)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
struct ioat_chan_common *chan = &ioat->base;
dma_addr_t phys_complete; dma_addr_t phys_complete;
u64 status; u64 status;
status = ioat_chansts(chan); status = ioat_chansts(ioat_chan);
/* when halted due to errors check for channel /* when halted due to errors check for channel
* programming errors before advancing the completion state * programming errors before advancing the completion state
@ -604,10 +596,10 @@ static void ioat3_timer_event(unsigned long data)
if (is_ioat_halted(status)) { if (is_ioat_halted(status)) {
u32 chanerr; u32 chanerr;
chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
dev_err(to_dev(chan), "%s: Channel halted (%x)\n", dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
__func__, chanerr); __func__, chanerr);
if (test_bit(IOAT_RUN, &chan->state)) if (test_bit(IOAT_RUN, &ioat_chan->state))
BUG_ON(is_ioat_bug(chanerr)); BUG_ON(is_ioat_bug(chanerr));
else /* we never got off the ground */ else /* we never got off the ground */
return; return;
@ -617,43 +609,43 @@ static void ioat3_timer_event(unsigned long data)
* acknowledged a pending completion once, then be more * acknowledged a pending completion once, then be more
* forceful with a restart * forceful with a restart
*/ */
spin_lock_bh(&chan->cleanup_lock); spin_lock_bh(&ioat_chan->cleanup_lock);
if (ioat_cleanup_preamble(chan, &phys_complete)) if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
__cleanup(ioat, phys_complete); __cleanup(ioat_chan, phys_complete);
else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
ioat3_restart_channel(ioat); ioat3_restart_channel(ioat_chan);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
spin_unlock_bh(&chan->cleanup_lock); spin_unlock_bh(&ioat_chan->cleanup_lock);
return; return;
} else { } else {
set_bit(IOAT_COMPLETION_ACK, &chan->state); set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
} }
if (ioat2_ring_active(ioat)) if (ioat2_ring_active(ioat_chan))
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
else { else {
spin_lock_bh(&ioat->prep_lock); spin_lock_bh(&ioat_chan->prep_lock);
check_active(ioat); check_active(ioat_chan);
spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&ioat_chan->prep_lock);
} }
spin_unlock_bh(&chan->cleanup_lock); spin_unlock_bh(&ioat_chan->cleanup_lock);
} }
static enum dma_status static enum dma_status
ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie, ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie,
struct dma_tx_state *txstate) struct dma_tx_state *txstate)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
enum dma_status ret; enum dma_status ret;
ret = dma_cookie_status(c, cookie, txstate); ret = dma_cookie_status(c, cookie, txstate);
if (ret == DMA_COMPLETE) if (ret == DMA_COMPLETE)
return ret; return ret;
ioat3_cleanup(ioat); ioat3_cleanup(ioat_chan);
return dma_cookie_status(c, cookie, txstate); return dma_cookie_status(c, cookie, txstate);
} }
@ -663,7 +655,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
size_t len, unsigned long flags) size_t len, unsigned long flags)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_ring_ent *compl_desc; struct ioat_ring_ent *compl_desc;
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
struct ioat_ring_ent *ext; struct ioat_ring_ent *ext;
@ -677,7 +669,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
BUG_ON(src_cnt < 2); BUG_ON(src_cnt < 2);
num_descs = ioat2_xferlen_to_descs(ioat, len); num_descs = ioat2_xferlen_to_descs(ioat_chan, len);
/* we need 2x the number of descriptors to cover greater than 5 /* we need 2x the number of descriptors to cover greater than 5
* sources * sources
*/ */
@ -692,24 +684,26 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
* (legacy) descriptor to ensure all completion writes arrive in * (legacy) descriptor to ensure all completion writes arrive in
* order. * order.
*/ */
if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0) if (likely(num_descs) &&
idx = ioat->head; ioat2_check_space_lock(ioat_chan, num_descs+1) == 0)
idx = ioat_chan->head;
else else
return NULL; return NULL;
i = 0; i = 0;
do { do {
struct ioat_raw_descriptor *descs[2]; struct ioat_raw_descriptor *descs[2];
size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); size_t xfer_size = min_t(size_t,
len, 1 << ioat_chan->xfercap_log);
int s; int s;
desc = ioat2_get_ring_ent(ioat, idx + i); desc = ioat2_get_ring_ent(ioat_chan, idx + i);
xor = desc->xor; xor = desc->xor;
/* save a branch by unconditionally retrieving the /* save a branch by unconditionally retrieving the
* extended descriptor xor_set_src() knows to not write * extended descriptor xor_set_src() knows to not write
* to it in the single descriptor case * to it in the single descriptor case
*/ */
ext = ioat2_get_ring_ent(ioat, idx + i + 1); ext = ioat2_get_ring_ent(ioat_chan, idx + i + 1);
xor_ex = ext->xor_ex; xor_ex = ext->xor_ex;
descs[0] = (struct ioat_raw_descriptor *) xor; descs[0] = (struct ioat_raw_descriptor *) xor;
@ -724,7 +718,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
len -= xfer_size; len -= xfer_size;
offset += xfer_size; offset += xfer_size;
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
} while ((i += 1 + with_ext) < num_descs); } while ((i += 1 + with_ext) < num_descs);
/* last xor descriptor carries the unmap parameters and fence bit */ /* last xor descriptor carries the unmap parameters and fence bit */
@ -735,7 +729,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
/* completion descriptor carries interrupt bit */ /* completion descriptor carries interrupt bit */
compl_desc = ioat2_get_ring_ent(ioat, idx + i); compl_desc = ioat2_get_ring_ent(ioat_chan, idx + i);
compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
hw = compl_desc->hw; hw = compl_desc->hw;
hw->ctl = 0; hw->ctl = 0;
@ -743,7 +737,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.compl_write = 1; hw->ctl_f.compl_write = 1;
hw->size = NULL_DESC_BUFFER_SIZE; hw->size = NULL_DESC_BUFFER_SIZE;
dump_desc_dbg(ioat, compl_desc); dump_desc_dbg(ioat_chan, compl_desc);
/* we leave the channel locked to ensure in order submission */ /* we leave the channel locked to ensure in order submission */
return &compl_desc->txd; return &compl_desc->txd;
@ -771,9 +765,10 @@ ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
} }
static void static void
dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext) dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc,
struct ioat_ring_ent *ext)
{ {
struct device *dev = to_dev(&ioat->base); struct device *dev = to_dev(ioat_chan);
struct ioat_pq_descriptor *pq = desc->pq; struct ioat_pq_descriptor *pq = desc->pq;
struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL; struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex }; struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
@ -797,10 +792,10 @@ dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct
dev_dbg(dev, "\tNEXT: %#llx\n", pq->next); dev_dbg(dev, "\tNEXT: %#llx\n", pq->next);
} }
static void dump_pq16_desc_dbg(struct ioat2_dma_chan *ioat, static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan,
struct ioat_ring_ent *desc) struct ioat_ring_ent *desc)
{ {
struct device *dev = to_dev(&ioat->base); struct device *dev = to_dev(ioat_chan);
struct ioat_pq_descriptor *pq = desc->pq; struct ioat_pq_descriptor *pq = desc->pq;
struct ioat_raw_descriptor *descs[] = { (void *)pq, struct ioat_raw_descriptor *descs[] = { (void *)pq,
(void *)pq, (void *)pq,
@ -838,9 +833,8 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
unsigned int src_cnt, const unsigned char *scf, unsigned int src_cnt, const unsigned char *scf,
size_t len, unsigned long flags) size_t len, unsigned long flags)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_chan_common *chan = &ioat->base; struct ioatdma_device *device = ioat_chan->device;
struct ioatdma_device *device = chan->device;
struct ioat_ring_ent *compl_desc; struct ioat_ring_ent *compl_desc;
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
struct ioat_ring_ent *ext; struct ioat_ring_ent *ext;
@ -853,13 +847,13 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
int i, s, idx, with_ext, num_descs; int i, s, idx, with_ext, num_descs;
int cb32 = (device->version < IOAT_VER_3_3) ? 1 : 0; int cb32 = (device->version < IOAT_VER_3_3) ? 1 : 0;
dev_dbg(to_dev(chan), "%s\n", __func__); dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
/* the engine requires at least two sources (we provide /* the engine requires at least two sources (we provide
* at least 1 implied source in the DMA_PREP_CONTINUE case) * at least 1 implied source in the DMA_PREP_CONTINUE case)
*/ */
BUG_ON(src_cnt + dmaf_continue(flags) < 2); BUG_ON(src_cnt + dmaf_continue(flags) < 2);
num_descs = ioat2_xferlen_to_descs(ioat, len); num_descs = ioat2_xferlen_to_descs(ioat_chan, len);
/* we need 2x the number of descriptors to cover greater than 3 /* we need 2x the number of descriptors to cover greater than 3
* sources (we need 1 extra source in the q-only continuation * sources (we need 1 extra source in the q-only continuation
* case and 3 extra sources in the p+q continuation case. * case and 3 extra sources in the p+q continuation case.
@ -877,23 +871,24 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
* order. * order.
*/ */
if (likely(num_descs) && if (likely(num_descs) &&
ioat2_check_space_lock(ioat, num_descs + cb32) == 0) ioat2_check_space_lock(ioat_chan, num_descs + cb32) == 0)
idx = ioat->head; idx = ioat_chan->head;
else else
return NULL; return NULL;
i = 0; i = 0;
do { do {
struct ioat_raw_descriptor *descs[2]; struct ioat_raw_descriptor *descs[2];
size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); size_t xfer_size = min_t(size_t, len,
1 << ioat_chan->xfercap_log);
desc = ioat2_get_ring_ent(ioat, idx + i); desc = ioat2_get_ring_ent(ioat_chan, idx + i);
pq = desc->pq; pq = desc->pq;
/* save a branch by unconditionally retrieving the /* save a branch by unconditionally retrieving the
* extended descriptor pq_set_src() knows to not write * extended descriptor pq_set_src() knows to not write
* to it in the single descriptor case * to it in the single descriptor case
*/ */
ext = ioat2_get_ring_ent(ioat, idx + i + with_ext); ext = ioat2_get_ring_ent(ioat_chan, idx + i + with_ext);
pq_ex = ext->pq_ex; pq_ex = ext->pq_ex;
descs[0] = (struct ioat_raw_descriptor *) pq; descs[0] = (struct ioat_raw_descriptor *) pq;
@ -932,7 +927,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
if (result) if (result)
desc->result = result; desc->result = result;
pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
dump_pq_desc_dbg(ioat, desc, ext); dump_pq_desc_dbg(ioat_chan, desc, ext);
if (!cb32) { if (!cb32) {
pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
@ -940,7 +935,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
compl_desc = desc; compl_desc = desc;
} else { } else {
/* completion descriptor carries interrupt bit */ /* completion descriptor carries interrupt bit */
compl_desc = ioat2_get_ring_ent(ioat, idx + i); compl_desc = ioat2_get_ring_ent(ioat_chan, idx + i);
compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
hw = compl_desc->hw; hw = compl_desc->hw;
hw->ctl = 0; hw->ctl = 0;
@ -948,7 +943,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.compl_write = 1; hw->ctl_f.compl_write = 1;
hw->size = NULL_DESC_BUFFER_SIZE; hw->size = NULL_DESC_BUFFER_SIZE;
dump_desc_dbg(ioat, compl_desc); dump_desc_dbg(ioat_chan, compl_desc);
} }
@ -962,9 +957,8 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
unsigned int src_cnt, const unsigned char *scf, unsigned int src_cnt, const unsigned char *scf,
size_t len, unsigned long flags) size_t len, unsigned long flags)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_chan_common *chan = &ioat->base; struct ioatdma_device *device = ioat_chan->device;
struct ioatdma_device *device = chan->device;
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
size_t total_len = len; size_t total_len = len;
struct ioat_pq_descriptor *pq; struct ioat_pq_descriptor *pq;
@ -975,16 +969,16 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
/* this function is only called with 9-16 sources */ /* this function is only called with 9-16 sources */
op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S;
dev_dbg(to_dev(chan), "%s\n", __func__); dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
num_descs = ioat2_xferlen_to_descs(ioat, len); num_descs = ioat2_xferlen_to_descs(ioat_chan, len);
/* /*
* 16 source pq is only available on cb3.3 and has no completion * 16 source pq is only available on cb3.3 and has no completion
* write hw bug. * write hw bug.
*/ */
if (num_descs && ioat2_check_space_lock(ioat, num_descs) == 0) if (num_descs && ioat2_check_space_lock(ioat_chan, num_descs) == 0)
idx = ioat->head; idx = ioat_chan->head;
else else
return NULL; return NULL;
@ -992,16 +986,17 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
do { do {
struct ioat_raw_descriptor *descs[4]; struct ioat_raw_descriptor *descs[4];
size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); size_t xfer_size = min_t(size_t, len,
1 << ioat_chan->xfercap_log);
desc = ioat2_get_ring_ent(ioat, idx + i); desc = ioat2_get_ring_ent(ioat_chan, idx + i);
pq = desc->pq; pq = desc->pq;
descs[0] = (struct ioat_raw_descriptor *) pq; descs[0] = (struct ioat_raw_descriptor *) pq;
desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3); desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3);
if (!desc->sed) { if (!desc->sed) {
dev_err(to_dev(chan), dev_err(to_dev(ioat_chan),
"%s: no free sed entries\n", __func__); "%s: no free sed entries\n", __func__);
return NULL; return NULL;
} }
@ -1051,7 +1046,7 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
pq->ctl_f.compl_write = 1; pq->ctl_f.compl_write = 1;
dump_pq16_desc_dbg(ioat, desc); dump_pq16_desc_dbg(ioat_chan, desc);
/* we leave the channel locked to ensure in order submission */ /* we leave the channel locked to ensure in order submission */
return &desc->txd; return &desc->txd;
@ -1177,12 +1172,12 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
static struct dma_async_tx_descriptor * static struct dma_async_tx_descriptor *
ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
{ {
struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
struct ioat_dma_descriptor *hw; struct ioat_dma_descriptor *hw;
if (ioat2_check_space_lock(ioat, 1) == 0) if (ioat2_check_space_lock(ioat_chan, 1) == 0)
desc = ioat2_get_ring_ent(ioat, ioat->head); desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head);
else else
return NULL; return NULL;
@ -1199,7 +1194,7 @@ ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
desc->txd.flags = flags; desc->txd.flags = flags;
desc->len = 1; desc->len = 1;
dump_desc_dbg(ioat, desc); dump_desc_dbg(ioat_chan, desc);
/* we leave the channel locked to ensure in order submission */ /* we leave the channel locked to ensure in order submission */
return &desc->txd; return &desc->txd;
@ -1504,10 +1499,10 @@ static int ioat3_irq_reinit(struct ioatdma_device *device)
case IOAT_MSIX: case IOAT_MSIX:
for (i = 0; i < device->common.chancnt; i++) { for (i = 0; i < device->common.chancnt; i++) {
struct msix_entry *msix = &device->msix_entries[i]; struct msix_entry *msix = &device->msix_entries[i];
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
chan = ioat_chan_by_index(device, i); ioat_chan = ioat_chan_by_index(device, i);
devm_free_irq(&pdev->dev, msix->vector, chan); devm_free_irq(&pdev->dev, msix->vector, ioat_chan);
} }
pci_disable_msix(pdev); pci_disable_msix(pdev);
@ -1526,21 +1521,21 @@ static int ioat3_irq_reinit(struct ioatdma_device *device)
return ioat_dma_setup_interrupts(device); return ioat_dma_setup_interrupts(device);
} }
static int ioat3_reset_hw(struct ioat_chan_common *chan) static int ioat3_reset_hw(struct ioatdma_chan *ioat_chan)
{ {
/* throw away whatever the channel was doing and get it /* throw away whatever the channel was doing and get it
* initialized, with ioat3 specific workarounds * initialized, with ioat3 specific workarounds
*/ */
struct ioatdma_device *device = chan->device; struct ioatdma_device *device = ioat_chan->device;
struct pci_dev *pdev = device->pdev; struct pci_dev *pdev = device->pdev;
u32 chanerr; u32 chanerr;
u16 dev_id; u16 dev_id;
int err; int err;
ioat2_quiesce(chan, msecs_to_jiffies(100)); ioat2_quiesce(ioat_chan, msecs_to_jiffies(100));
chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
if (device->version < IOAT_VER_3_3) { if (device->version < IOAT_VER_3_3) {
/* clear any pending errors */ /* clear any pending errors */
@ -1565,7 +1560,7 @@ static int ioat3_reset_hw(struct ioat_chan_common *chan)
} }
} }
err = ioat2_reset_sync(chan, msecs_to_jiffies(200)); err = ioat2_reset_sync(ioat_chan, msecs_to_jiffies(200));
if (!err) if (!err)
err = ioat3_irq_reinit(device); err = ioat3_irq_reinit(device);
@ -1579,7 +1574,7 @@ static void ioat3_intr_quirk(struct ioatdma_device *device)
{ {
struct dma_device *dma; struct dma_device *dma;
struct dma_chan *c; struct dma_chan *c;
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
u32 errmask; u32 errmask;
dma = &device->common; dma = &device->common;
@ -1590,12 +1585,12 @@ static void ioat3_intr_quirk(struct ioatdma_device *device)
*/ */
if (device->cap & IOAT_CAP_DWBES) { if (device->cap & IOAT_CAP_DWBES) {
list_for_each_entry(c, &dma->channels, device_node) { list_for_each_entry(c, &dma->channels, device_node) {
chan = to_chan_common(c); ioat_chan = to_ioat_chan(c);
errmask = readl(chan->reg_base + errmask = readl(ioat_chan->reg_base +
IOAT_CHANERR_MASK_OFFSET); IOAT_CHANERR_MASK_OFFSET);
errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR | errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR |
IOAT_CHANERR_XOR_Q_ERR; IOAT_CHANERR_XOR_Q_ERR;
writel(errmask, chan->reg_base + writel(errmask, ioat_chan->reg_base +
IOAT_CHANERR_MASK_OFFSET); IOAT_CHANERR_MASK_OFFSET);
} }
} }
@ -1607,7 +1602,7 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca)
int dca_en = system_has_dca_enabled(pdev); int dca_en = system_has_dca_enabled(pdev);
struct dma_device *dma; struct dma_device *dma;
struct dma_chan *c; struct dma_chan *c;
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
bool is_raid_device = false; bool is_raid_device = false;
int err; int err;
@ -1702,9 +1697,9 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca)
return err; return err;
list_for_each_entry(c, &dma->channels, device_node) { list_for_each_entry(c, &dma->channels, device_node) {
chan = to_chan_common(c); ioat_chan = to_ioat_chan(c);
writel(IOAT_DMA_DCA_ANY_CPU, writel(IOAT_DMA_DCA_ANY_CPU,
chan->reg_base + IOAT_DCACTRL_OFFSET); ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
} }
err = ioat_register(device); err = ioat_register(device);