ioat: switch watchdog and reset handler from workqueue to timer (09c8a5b8) · Commits · 戴 / test

drivers/dma/ioat/dma.c

+136 −215

Original line number	Diff line number	Diff line
		@@ -99,23 +99,26 @@ static void ioat1_cleanup_tasklet(unsigned long data);
		/* common channel initialization */
		void ioat_init_channel(struct ioatdma_device *device,
		struct ioat_chan_common *chan, int idx,
		work_func_t work_fn, void (*tasklet)(unsigned long),
		unsigned long tasklet_data)
		void (*timer_fn)(unsigned long),
		void (*tasklet)(unsigned long),
		unsigned long ioat)
		{
		struct dma_device *dma = &device->common;

		chan->device = device;
		chan->reg_base = device->reg_base + (0x80 * (idx + 1));
		INIT_DELAYED_WORK(&chan->work, work_fn);
		spin_lock_init(&chan->cleanup_lock);
		chan->common.device = dma;
		list_add_tail(&chan->common.device_node, &dma->channels);
		device->idx[idx] = chan;
		tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
		init_timer(&chan->timer);
		chan->timer.function = timer_fn;
		chan->timer.data = ioat;
		tasklet_init(&chan->cleanup_task, tasklet, ioat);
		tasklet_disable(&chan->cleanup_task);
		}

		static void ioat1_reset_part2(struct work_struct *work);
		static void ioat1_timer_event(unsigned long data);

		/**
		* ioat1_dma_enumerate_channels - find and initialize the device's channels
		@@ -153,7 +156,7 @@ static int ioat1_enumerate_channels(struct ioatdma_device *device)
		break;

		ioat_init_channel(device, &ioat->base, i,
		ioat1_reset_part2,
		ioat1_timer_event,
		ioat1_cleanup_tasklet,
		(unsigned long) ioat);
		ioat->xfercap = xfercap;
		@@ -192,61 +195,6 @@ static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
		}
		}

		/**
		* ioat1_reset_part2 - reinit the channel after a reset
		*/
		static void ioat1_reset_part2(struct work_struct *work)
		{
		struct ioat_chan_common *chan;
		struct ioat_dma_chan *ioat;
		struct ioat_desc_sw *desc;
		int dmacount;
		bool start_null = false;

		chan = container_of(work, struct ioat_chan_common, work.work);
		ioat = container_of(chan, struct ioat_dma_chan, base);
		spin_lock_bh(&chan->cleanup_lock);
		spin_lock_bh(&ioat->desc_lock);

		*chan->completion = 0;
		ioat->pending = 0;

		/* count the descriptors waiting */
		dmacount = 0;
		if (ioat->used_desc.prev) {
		desc = to_ioat_desc(ioat->used_desc.prev);
		do {
		dmacount++;
		desc = to_ioat_desc(desc->node.next);
		} while (&desc->node != ioat->used_desc.next);
		}

		if (dmacount) {
		/*
		* write the new starting descriptor address
		* this puts channel engine into ARMED state
		*/
		desc = to_ioat_desc(ioat->used_desc.prev);
		writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
		chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
		writel(((u64) desc->txd.phys) >> 32,
		chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);

		writeb(IOAT_CHANCMD_START, chan->reg_base
		+ IOAT_CHANCMD_OFFSET(chan->device->version));
		} else
		start_null = true;
		spin_unlock_bh(&ioat->desc_lock);
		spin_unlock_bh(&chan->cleanup_lock);

		dev_err(to_dev(chan),
		"chan%d reset - %d descs waiting, %d total desc\n",
		chan_num(chan), dmacount, ioat->desccount);

		if (start_null)
		ioat1_dma_start_null_desc(ioat);
		}

		/**
		* ioat1_reset_channel - restart a channel
		* @ioat: IOAT DMA channel handle
		@@ -257,12 +205,9 @@ static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
		void __iomem *reg_base = chan->reg_base;
		u32 chansts, chanerr;

		if (!ioat->used_desc.prev)
		return;

		dev_dbg(to_dev(chan), "%s\n", __func__);
		dev_warn(to_dev(chan), "reset\n");
		chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
		chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
		chansts = *chan->completion & IOAT_CHANSTS_STATUS;
		if (chanerr) {
		dev_err(to_dev(chan),
		"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
		@@ -278,93 +223,11 @@ static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
		* while we're waiting.
		*/

		spin_lock_bh(&ioat->desc_lock);
		ioat->pending = INT_MIN;
		writeb(IOAT_CHANCMD_RESET,
		reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
		spin_unlock_bh(&ioat->desc_lock);

		/* schedule the 2nd half instead of sleeping a long time */
		schedule_delayed_work(&chan->work, RESET_DELAY);
		}

		/**
		* ioat1_chan_watchdog - watch for stuck channels
		*/
		static void ioat1_chan_watchdog(struct work_struct *work)
		{
		struct ioatdma_device *device =
		container_of(work, struct ioatdma_device, work.work);
		struct ioat_dma_chan *ioat;
		struct ioat_chan_common *chan;
		int i;
		u64 completion;
		u32 completion_low;
		unsigned long compl_desc_addr_hw;

		for (i = 0; i < device->common.chancnt; i++) {
		chan = ioat_chan_by_index(device, i);
		ioat = container_of(chan, struct ioat_dma_chan, base);

		if (/* have we started processing anything yet */
		chan->last_completion
		/* have we completed any since last watchdog cycle? */
		&& (chan->last_completion == chan->watchdog_completion)
		/* has TCP stuck on one cookie since last watchdog? */
		&& (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
		&& (chan->watchdog_tcp_cookie != chan->completed_cookie)
		/* is there something in the chain to be processed? */
		/* CB1 chain always has at least the last one processed */
		&& (ioat->used_desc.prev != ioat->used_desc.next)
		&& ioat->pending == 0) {

		/*
		* check CHANSTS register for completed
		* descriptor address.
		* if it is different than completion writeback,
		* it is not zero
		* and it has changed since the last watchdog
		* we can assume that channel
		* is still working correctly
		* and the problem is in completion writeback.
		* update completion writeback
		* with actual CHANSTS value
		* else
		* try resetting the channel
		*/

		/* we need to read the low address first as this
		* causes the chipset to latch the upper bits
		* for the subsequent read
		*/
		completion_low = readl(chan->reg_base +
		IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
		completion = readl(chan->reg_base +
		IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
		completion <<= 32;
		completion \|= completion_low;
		compl_desc_addr_hw = completion &
		IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;

		if ((compl_desc_addr_hw != 0)
		&& (compl_desc_addr_hw != chan->watchdog_completion)
		&& (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
		chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
		*chan->completion = completion;
		} else {
		ioat1_reset_channel(ioat);
		chan->watchdog_completion = 0;
		chan->last_compl_desc_addr_hw = 0;
		}
		} else {
		chan->last_compl_desc_addr_hw = 0;
		chan->watchdog_completion = chan->last_completion;
		}

		chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
		}

		schedule_delayed_work(&device->work, WATCHDOG_DELAY);
		set_bit(IOAT_RESET_PENDING, &chan->state);
		mod_timer(&chan->timer, jiffies + RESET_DELAY);
		}

		static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
		@@ -372,6 +235,7 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
		struct dma_chan *c = tx->chan;
		struct ioat_dma_chan *ioat = to_ioat_chan(c);
		struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
		struct ioat_chan_common *chan = &ioat->base;
		struct ioat_desc_sw *first;
		struct ioat_desc_sw *chain_tail;
		dma_cookie_t cookie;
		@@ -396,6 +260,9 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
		dump_desc_dbg(ioat, chain_tail);
		dump_desc_dbg(ioat, first);

		if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
		mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);

		ioat->pending += desc->hw->tx_cnt;
		if (ioat->pending >= ioat_pending_level)
		__ioat1_dma_memcpy_issue_pending(ioat);
		@@ -520,6 +387,7 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c)
		return;

		tasklet_disable(&chan->cleanup_task);
		del_timer_sync(&chan->timer);
		ioat1_cleanup(ioat);

		/* Delay 100ms after reset to allow internal DMA logic to quiesce
		@@ -560,9 +428,6 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c)

		chan->last_completion = 0;
		chan->completion_dma = 0;
		chan->watchdog_completion = 0;
		chan->last_compl_desc_addr_hw = 0;
		chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
		ioat->pending = 0;
		ioat->desccount = 0;
		}
		@@ -705,15 +570,15 @@ unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
		u64 completion;

		completion = *chan->completion;
		phys_complete = completion & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
		phys_complete = ioat_chansts_to_addr(completion);

		dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
		(unsigned long long) phys_complete);

		if ((completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
		IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
		if (is_ioat_halted(completion)) {
		u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
		dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
		readl(chan->reg_base + IOAT_CHANERR_OFFSET));
		chanerr);

		/* TODO do something to salvage the situation */
		}
		@@ -721,48 +586,31 @@ unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
		return phys_complete;
		}

		/**
		* ioat1_cleanup - cleanup up finished descriptors
		* @chan: ioat channel to be cleaned up
		*/
		static void ioat1_cleanup(struct ioat_dma_chan *ioat)
		bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
		unsigned long *phys_complete)
		{
		struct ioat_chan_common *chan = &ioat->base;
		unsigned long phys_complete;
		struct ioat_desc_sw desc, _desc;
		dma_cookie_t cookie = 0;
		struct dma_async_tx_descriptor *tx;

		prefetch(chan->completion);
		*phys_complete = ioat_get_current_completion(chan);
		if (*phys_complete == chan->last_completion)
		return false;
		clear_bit(IOAT_COMPLETION_ACK, &chan->state);
		mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);

		if (!spin_trylock_bh(&chan->cleanup_lock))
		return;

		phys_complete = ioat_get_current_completion(chan);
		if (phys_complete == chan->last_completion) {
		spin_unlock_bh(&chan->cleanup_lock);
		/*
		* perhaps we're stuck so hard that the watchdog can't go off?
		* try to catch it after 2 seconds
		*/
		if (time_after(jiffies,
		chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
		ioat1_chan_watchdog(&(chan->device->work.work));
		chan->last_completion_time = jiffies;
		return true;
		}
		return;
		}
		chan->last_completion_time = jiffies;

		cookie = 0;
		if (!spin_trylock_bh(&ioat->desc_lock)) {
		spin_unlock_bh(&chan->cleanup_lock);
		return;
		}
		static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
		{
		struct ioat_chan_common *chan = &ioat->base;
		struct list_head _desc, n;
		struct dma_async_tx_descriptor *tx;

		dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
		__func__, phys_complete);
		list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
		list_for_each_safe(_desc, n, &ioat->used_desc) {
		struct ioat_desc_sw *desc;

		prefetch(n);
		desc = list_entry(_desc, typeof(*desc), node);
		tx = &desc->txd;
		/*
		* Incoming DMA requests may use multiple descriptors,
		@@ -771,7 +619,8 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat)
		*/
		dump_desc_dbg(ioat, desc);
		if (tx->cookie) {
		cookie = tx->cookie;
		chan->completed_cookie = tx->cookie;
		tx->cookie = 0;
		ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
		if (tx->callback) {
		tx->callback(tx->callback_param);
		@@ -786,27 +635,110 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat)
		*/
		if (async_tx_test_ack(tx))
		list_move_tail(&desc->node, &ioat->free_desc);
		else
		tx->cookie = 0;
		} else {
		/*
		* last used desc. Do not remove, so we can
		* append from it, but don't look at it next
		* time, either
		* append from it.
		*/
		tx->cookie = 0;

		/* if nothing else is pending, cancel the
		* completion timeout
		*/
		if (n == &ioat->used_desc) {
		dev_dbg(to_dev(chan),
		"%s cancel completion timeout\n",
		__func__);
		clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
		}

		/* TODO check status bits? */
		break;
		}
		}

		chan->last_completion = phys_complete;
		}

		/**
		* ioat1_cleanup - cleanup up finished descriptors
		* @chan: ioat channel to be cleaned up
		*
		* To prevent lock contention we defer cleanup when the locks are
		* contended with a terminal timeout that forces cleanup and catches
		* completion notification errors.
		*/
		static void ioat1_cleanup(struct ioat_dma_chan *ioat)
		{
		struct ioat_chan_common *chan = &ioat->base;
		unsigned long phys_complete;

		prefetch(chan->completion);

		if (!spin_trylock_bh(&chan->cleanup_lock))
		return;

		if (!ioat_cleanup_preamble(chan, &phys_complete)) {
		spin_unlock_bh(&chan->cleanup_lock);
		return;
		}

		if (!spin_trylock_bh(&ioat->desc_lock)) {
		spin_unlock_bh(&chan->cleanup_lock);
		return;
		}

		__cleanup(ioat, phys_complete);

		spin_unlock_bh(&ioat->desc_lock);
		spin_unlock_bh(&chan->cleanup_lock);
		}

		chan->last_completion = phys_complete;
		if (cookie != 0)
		chan->completed_cookie = cookie;
		static void ioat1_timer_event(unsigned long data)
		{
		struct ioat_dma_chan ioat = (void ) data;
		struct ioat_chan_common *chan = &ioat->base;

		dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);

		spin_lock_bh(&chan->cleanup_lock);
		if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
		struct ioat_desc_sw *desc;

		spin_lock_bh(&ioat->desc_lock);

		/* restart active descriptors */
		desc = to_ioat_desc(ioat->used_desc.prev);
		ioat_set_chainaddr(ioat, desc->txd.phys);
		ioat_start(chan);

		ioat->pending = 0;
		set_bit(IOAT_COMPLETION_PENDING, &chan->state);
		mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
		spin_unlock_bh(&ioat->desc_lock);
		} else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
		unsigned long phys_complete;

		spin_lock_bh(&ioat->desc_lock);
		/* if we haven't made progress and we have already
		* acknowledged a pending completion once, then be more
		* forceful with a restart
		*/
		if (ioat_cleanup_preamble(chan, &phys_complete))
		__cleanup(ioat, phys_complete);
		else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
		ioat1_reset_channel(ioat);
		else {
		u64 status = ioat_chansts(chan);

		/* manually update the last completion address */
		if (ioat_chansts_to_addr(status) != 0)
		*chan->completion = status;

		set_bit(IOAT_COMPLETION_ACK, &chan->state);
		mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
		}
		spin_unlock_bh(&ioat->desc_lock);
		}
		spin_unlock_bh(&chan->cleanup_lock);
		}

		@@ -855,13 +787,8 @@ static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
		list_add_tail(&desc->node, &ioat->used_desc);
		dump_desc_dbg(ioat, desc);

		writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
		chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
		writel(((u64) desc->txd.phys) >> 32,
		chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);

		writeb(IOAT_CHANCMD_START, chan->reg_base
		+ IOAT_CHANCMD_OFFSET(chan->device->version));
		ioat_set_chainaddr(ioat, desc->txd.phys);
		ioat_start(chan);
		spin_unlock_bh(&ioat->desc_lock);
		}

		@@ -1194,9 +1121,6 @@ int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
		if (dca)
		device->dca = ioat_dca_init(pdev, device->reg_base);

		INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
		schedule_delayed_work(&device->work, WATCHDOG_DELAY);

		return err;
		}

		@@ -1204,9 +1128,6 @@ void __devexit ioat_dma_remove(struct ioatdma_device *device)
		{
		struct dma_device *dma = &device->common;

		if (device->version != IOAT_VER_3_0)
		cancel_delayed_work(&device->work);

		ioat_disable_interrupts(device);

		dma_async_device_unregister(dma);

drivers/dma/ioat/dma.h

+93 −19

Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@

		#include <linux/dmaengine.h>
		#include "hw.h"
		#include "registers.h"
		#include <linux/init.h>
		#include <linux/dmapool.h>
		#include <linux/cache.h>
		@@ -33,7 +34,6 @@

		#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
		#define IOAT_DMA_DCA_ANY_CPU ~0
		#define IOAT_WATCHDOG_PERIOD (2 * HZ)

		#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
		#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
		@@ -42,9 +42,6 @@

		#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)

		#define RESET_DELAY msecs_to_jiffies(100)
		#define WATCHDOG_DELAY round_jiffies(msecs_to_jiffies(2000))

		/*
		* workaround for IOAT ver.3.0 null descriptor issue
		* (channel returns error when size is 0)
		@@ -72,7 +69,6 @@ struct ioatdma_device {
		struct pci_pool *completion_pool;
		struct dma_device common;
		u8 version;
		struct delayed_work work;
		struct msix_entry msix_entries[4];
		struct ioat_chan_common *idx[4];
		struct dca_provider *dca;
		@@ -81,24 +77,21 @@ struct ioatdma_device {
		};

		struct ioat_chan_common {
		struct dma_chan common;
		void __iomem *reg_base;

		unsigned long last_completion;
		unsigned long last_completion_time;

		spinlock_t cleanup_lock;
		dma_cookie_t completed_cookie;
		unsigned long watchdog_completion;
		int watchdog_tcp_cookie;
		u32 watchdog_last_tcp_cookie;
		struct delayed_work work;

		unsigned long state;
		#define IOAT_COMPLETION_PENDING 0
		#define IOAT_COMPLETION_ACK 1
		#define IOAT_RESET_PENDING 2
		struct timer_list timer;
		#define COMPLETION_TIMEOUT msecs_to_jiffies(100)
		#define RESET_DELAY msecs_to_jiffies(100)
		struct ioatdma_device *device;
		struct dma_chan common;

		dma_addr_t completion_dma;
		u64 *completion;
		unsigned long last_compl_desc_addr_hw;
		struct tasklet_struct cleanup_task;
		};

		@@ -148,7 +141,6 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,

		last_used = c->cookie;
		last_complete = chan->completed_cookie;
		chan->watchdog_tcp_cookie = cookie;

		if (done)
		*done = last_complete;
		@@ -215,6 +207,85 @@ ioat_chan_by_index(struct ioatdma_device *device, int index)
		return device->idx[index];
		}

		static inline u64 ioat_chansts(struct ioat_chan_common *chan)
		{
		u8 ver = chan->device->version;
		u64 status;
		u32 status_lo;

		/* We need to read the low address first as this causes the
		* chipset to latch the upper bits for the subsequent read
		*/
		status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
		status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
		status <<= 32;
		status \|= status_lo;

		return status;
		}

		static inline void ioat_start(struct ioat_chan_common *chan)
		{
		u8 ver = chan->device->version;

		writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
		}

		static inline u64 ioat_chansts_to_addr(u64 status)
		{
		return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
		}

		static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
		{
		return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
		}

		static inline void ioat_suspend(struct ioat_chan_common *chan)
		{
		u8 ver = chan->device->version;

		writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
		}

		static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
		{
		struct ioat_chan_common *chan = &ioat->base;

		writel(addr & 0x00000000FFFFFFFF,
		chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
		writel(addr >> 32,
		chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
		}

		static inline bool is_ioat_active(unsigned long status)
		{
		return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
		}

		static inline bool is_ioat_idle(unsigned long status)
		{
		return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
		}

		static inline bool is_ioat_halted(unsigned long status)
		{
		return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
		}

		static inline bool is_ioat_suspended(unsigned long status)
		{
		return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
		}

		/* channel was fatally programmed */
		static inline bool is_ioat_bug(unsigned long err)
		{
		return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR\|IOAT_CHANERR_DEST_ADDR_ERR\|
		IOAT_CHANERR_NEXT_ADDR_ERR\|IOAT_CHANERR_CONTROL_ERR\|
		IOAT_CHANERR_LENGTH_ERR));
		}

		int __devinit ioat_probe(struct ioatdma_device *device);
		int __devinit ioat_register(struct ioatdma_device *device);
		int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
		@@ -224,8 +295,11 @@ struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
		unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
		void ioat_init_channel(struct ioatdma_device *device,
		struct ioat_chan_common *chan, int idx,
		work_func_t work_fn, void (*tasklet)(unsigned long),
		unsigned long tasklet_data);
		void (*timer_fn)(unsigned long),
		void (*tasklet)(unsigned long),
		unsigned long ioat);
		void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
		size_t len, struct ioat_dma_descriptor *hw);
		bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
		unsigned long *phys_complete);
		#endif /* IOATDMA_H */

drivers/dma/ioat/dma_v2.c

+138 −183

File changed.

Preview size limit exceeded, changes collapsed.

drivers/dma/ioat/dma_v2.h

+10 −0

Original line number	Diff line number	Diff line
		@@ -127,6 +127,16 @@ ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
		return ioat->ring[idx & ioat2_ring_mask(ioat)];
		}

		static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
		{
		struct ioat_chan_common *chan = &ioat->base;

		writel(addr & 0x00000000FFFFFFFF,
		chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
		writel(addr >> 32,
		chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
		}

		int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
		int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
		struct dca_provider * __devinit ioat2_dca_init(struct pci_dev pdev, void __iomem iobase);

drivers/dma/ioat/registers.h

+11 −11

Original line number	Diff line number	Diff line
		@@ -101,11 +101,11 @@
		#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR (~0x3fULL)
		#define IOAT_CHANSTS_SOFT_ERR 0x10ULL
		#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x8ULL
		#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x7ULL
		#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
		#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
		#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
		#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
		#define IOAT_CHANSTS_STATUS 0x7ULL
		#define IOAT_CHANSTS_ACTIVE 0x0
		#define IOAT_CHANSTS_DONE 0x1
		#define IOAT_CHANSTS_SUSPENDED 0x2
		#define IOAT_CHANSTS_HALTED 0x3



		@@ -208,18 +208,18 @@
		#define IOAT_CDAR_OFFSET_HIGH 0x24

		#define IOAT_CHANERR_OFFSET 0x28 /* 32-bit Channel Error Register */
		#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR 0x0001
		#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR 0x0002
		#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR 0x0004
		#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR 0x0008
		#define IOAT_CHANERR_SRC_ADDR_ERR 0x0001
		#define IOAT_CHANERR_DEST_ADDR_ERR 0x0002
		#define IOAT_CHANERR_NEXT_ADDR_ERR 0x0004
		#define IOAT_CHANERR_NEXT_DESC_ALIGN_ERR 0x0008
		#define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR 0x0010
		#define IOAT_CHANERR_CHANCMD_ERR 0x0020
		#define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0040
		#define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0080
		#define IOAT_CHANERR_READ_DATA_ERR 0x0100
		#define IOAT_CHANERR_WRITE_DATA_ERR 0x0200
		#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR 0x0400
		#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR 0x0800
		#define IOAT_CHANERR_CONTROL_ERR 0x0400
		#define IOAT_CHANERR_LENGTH_ERR 0x0800
		#define IOAT_CHANERR_COMPLETION_ADDR_ERR 0x1000
		#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000
		#define IOAT_CHANERR_SOFT_ERR 0x4000

Admin message