Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx (07fe944e) · Commits · 戴 / test

crypto/async_tx/async_memcpy.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -77,7 +77,7 @@ async_memcpy(struct page dest, struct page src, unsigned int dest_offset,
	/* if ack is already set then we cannot be sure		/* if ack is already set then we cannot be sure
	* we are referring to the correct operation		* we are referring to the correct operation
	*/		*/
	BUG_ON(depend_tx->ack);		BUG_ON(async_tx_test_ack(depend_tx));
	if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)		if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
	panic("%s: DMA_ERROR waiting for depend_tx\n",		panic("%s: DMA_ERROR waiting for depend_tx\n",
	__func__);		__func__);

crypto/async_tx/async_tx.c

+165 −37

Original line number	Original line	Diff line number	Diff line
	@@ -89,13 +89,19 @@ dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
	iter = tx;		iter = tx;

	/* find the root of the unsubmitted dependency chain */		/* find the root of the unsubmitted dependency chain */
	while (iter->cookie == -EBUSY) {		do {
	parent = iter->parent;		parent = iter->parent;
	if (parent && parent->cookie == -EBUSY)		if (!parent)
	iter = iter->parent;
	else
	break;		break;
	}		else
			iter = parent;
			} while (parent);

			/* there is a small window for ->parent == NULL and
			* ->cookie == -EBUSY
			*/
			while (iter->cookie == -EBUSY)
			cpu_relax();

	status = dma_sync_wait(iter->chan, iter->cookie);		status = dma_sync_wait(iter->chan, iter->cookie);
	} while (status == DMA_IN_PROGRESS \|\| (iter != tx));		} while (status == DMA_IN_PROGRESS \|\| (iter != tx));
	@@ -111,24 +117,33 @@ EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
	void		void
	async_tx_run_dependencies(struct dma_async_tx_descriptor *tx)		async_tx_run_dependencies(struct dma_async_tx_descriptor *tx)
	{		{
	struct dma_async_tx_descriptor dep_tx, _dep_tx;		struct dma_async_tx_descriptor *next = tx->next;
	struct dma_device *dev;
	struct dma_chan *chan;		struct dma_chan *chan;

	list_for_each_entry_safe(dep_tx, _dep_tx, &tx->depend_list,		if (!next)
	depend_node) {		return;
	chan = dep_tx->chan;
	dev = chan->device;		tx->next = NULL;
	/* we can't depend on ourselves */		chan = next->chan;
	BUG_ON(chan == tx->chan);
	list_del(&dep_tx->depend_node);		/* keep submitting up until a channel switch is detected
	tx->tx_submit(dep_tx);		* in that case we will be called again as a result of
			* processing the interrupt from async_tx_channel_switch
	/* we need to poke the engine as client code does not
	* know about dependency submission events
	*/		*/
	dev->device_issue_pending(chan);		while (next && next->chan == chan) {
			struct dma_async_tx_descriptor *_next;

			spin_lock_bh(&next->lock);
			next->parent = NULL;
			_next = next->next;
			next->next = NULL;
			spin_unlock_bh(&next->lock);

			next->tx_submit(next);
			next = _next;
	}		}

			chan->device->device_issue_pending(chan);
	}		}
	EXPORT_SYMBOL_GPL(async_tx_run_dependencies);		EXPORT_SYMBOL_GPL(async_tx_run_dependencies);

	@@ -397,6 +412,92 @@ static void __exit async_tx_exit(void)
	}		}
	#endif		#endif


			/**
			* async_tx_channel_switch - queue an interrupt descriptor with a dependency
			* pre-attached.
			* @depend_tx: the operation that must finish before the new operation runs
			* @tx: the new operation
			*/
			static void
			async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
			struct dma_async_tx_descriptor *tx)
			{
			struct dma_chan *chan;
			struct dma_device *device;
			struct dma_async_tx_descriptor intr_tx = (void ) ~0;

			/* first check to see if we can still append to depend_tx */
			spin_lock_bh(&depend_tx->lock);
			if (depend_tx->parent && depend_tx->chan == tx->chan) {
			tx->parent = depend_tx;
			depend_tx->next = tx;
			intr_tx = NULL;
			}
			spin_unlock_bh(&depend_tx->lock);

			if (!intr_tx)
			return;

			chan = depend_tx->chan;
			device = chan->device;

			/* see if we can schedule an interrupt
			* otherwise poll for completion
			*/
			if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
			intr_tx = device->device_prep_dma_interrupt(chan, 0);
			else
			intr_tx = NULL;

			if (intr_tx) {
			intr_tx->callback = NULL;
			intr_tx->callback_param = NULL;
			tx->parent = intr_tx;
			/* safe to set ->next outside the lock since we know we are
			* not submitted yet
			*/
			intr_tx->next = tx;

			/* check if we need to append */
			spin_lock_bh(&depend_tx->lock);
			if (depend_tx->parent) {
			intr_tx->parent = depend_tx;
			depend_tx->next = intr_tx;
			async_tx_ack(intr_tx);
			intr_tx = NULL;
			}
			spin_unlock_bh(&depend_tx->lock);

			if (intr_tx) {
			intr_tx->parent = NULL;
			intr_tx->tx_submit(intr_tx);
			async_tx_ack(intr_tx);
			}
			} else {
			if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
			panic("%s: DMA_ERROR waiting for depend_tx\n",
			__func__);
			tx->tx_submit(tx);
			}
			}


			/**
			* submit_disposition - while holding depend_tx->lock we must avoid submitting
			* new operations to prevent a circular locking dependency with
			* drivers that already hold a channel lock when calling
			* async_tx_run_dependencies.
			* @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
			* @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
			* @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
			*/
			enum submit_disposition {
			ASYNC_TX_SUBMITTED,
			ASYNC_TX_CHANNEL_SWITCH,
			ASYNC_TX_DIRECT_SUBMIT,
			};

	void		void
	async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,		async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,
	enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,		enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,
	@@ -405,28 +506,55 @@ async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,
	tx->callback = cb_fn;		tx->callback = cb_fn;
	tx->callback_param = cb_param;		tx->callback_param = cb_param;

	/* set this new tx to run after depend_tx if:		if (depend_tx) {
	* 1/ a dependency exists (depend_tx is !NULL)		enum submit_disposition s;
	* 2/ the tx can not be submitted to the current channel
	*/		/* sanity check the dependency chain:
	if (depend_tx && depend_tx->chan != chan) {		* 1/ if ack is already set then we cannot be sure
	/* if ack is already set then we cannot be sure
	* we are referring to the correct operation		* we are referring to the correct operation
			* 2/ dependencies are 1:1 i.e. two transactions can
			* not depend on the same parent
	*/		*/
	BUG_ON(depend_tx->ack);		BUG_ON(async_tx_test_ack(depend_tx) \|\| depend_tx->next \|\|
			tx->parent);

	tx->parent = depend_tx;		/* the lock prevents async_tx_run_dependencies from missing
			* the setting of ->next when ->parent != NULL
			*/
	spin_lock_bh(&depend_tx->lock);		spin_lock_bh(&depend_tx->lock);
	list_add_tail(&tx->depend_node, &depend_tx->depend_list);		if (depend_tx->parent) {
	if (depend_tx->cookie == 0) {		/* we have a parent so we can not submit directly
	struct dma_chan *dep_chan = depend_tx->chan;		* if we are staying on the same channel: append
	struct dma_device *dep_dev = dep_chan->device;		* else: channel switch
	dep_dev->device_dependency_added(dep_chan);		*/
			if (depend_tx->chan == chan) {
			tx->parent = depend_tx;
			depend_tx->next = tx;
			s = ASYNC_TX_SUBMITTED;
			} else
			s = ASYNC_TX_CHANNEL_SWITCH;
			} else {
			/* we do not have a parent so we may be able to submit
			* directly if we are staying on the same channel
			*/
			if (depend_tx->chan == chan)
			s = ASYNC_TX_DIRECT_SUBMIT;
			else
			s = ASYNC_TX_CHANNEL_SWITCH;
	}		}
	spin_unlock_bh(&depend_tx->lock);		spin_unlock_bh(&depend_tx->lock);

	/* schedule an interrupt to trigger the channel switch */		switch (s) {
	async_trigger_callback(ASYNC_TX_ACK, depend_tx, NULL, NULL);		case ASYNC_TX_SUBMITTED:
			break;
			case ASYNC_TX_CHANNEL_SWITCH:
			async_tx_channel_switch(depend_tx, tx);
			break;
			case ASYNC_TX_DIRECT_SUBMIT:
			tx->parent = NULL;
			tx->tx_submit(tx);
			break;
			}
	} else {		} else {
	tx->parent = NULL;		tx->parent = NULL;
	tx->tx_submit(tx);		tx->tx_submit(tx);
	@@ -467,7 +595,7 @@ async_trigger_callback(enum async_tx_flags flags,
	if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))		if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))
	device = NULL;		device = NULL;

	tx = device ? device->device_prep_dma_interrupt(chan) : NULL;		tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;
	} else		} else
	tx = NULL;		tx = NULL;

	@@ -483,7 +611,7 @@ async_trigger_callback(enum async_tx_flags flags,
	/* if ack is already set then we cannot be sure		/* if ack is already set then we cannot be sure
	* we are referring to the correct operation		* we are referring to the correct operation
	*/		*/
	BUG_ON(depend_tx->ack);		BUG_ON(async_tx_test_ack(depend_tx));
	if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)		if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
	panic("%s: DMA_ERROR waiting for depend_tx\n",		panic("%s: DMA_ERROR waiting for depend_tx\n",
	__func__);		__func__);

crypto/async_tx/async_xor.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -191,7 +191,7 @@ async_xor(struct page dest, struct page *src_list, unsigned int offset,
	/* if ack is already set then we cannot be sure		/* if ack is already set then we cannot be sure
	* we are referring to the correct operation		* we are referring to the correct operation
	*/		*/
	BUG_ON(depend_tx->ack);		BUG_ON(async_tx_test_ack(depend_tx));
	if (dma_wait_for_async_tx(depend_tx) ==		if (dma_wait_for_async_tx(depend_tx) ==
	DMA_ERROR)		DMA_ERROR)
	panic("%s: DMA_ERROR waiting for "		panic("%s: DMA_ERROR waiting for "

drivers/dma/Kconfig

+0 −8

Original line number	Original line	Diff line number	Diff line
	@@ -46,14 +46,6 @@ config FSL_DMA
	MPC8560/40, MPC8555, MPC8548 and MPC8641 processors.		MPC8560/40, MPC8555, MPC8548 and MPC8641 processors.
	The MPC8349, MPC8360 is also supported.		The MPC8349, MPC8360 is also supported.

	config FSL_DMA_SELFTEST
	bool "Enable the self test for each DMA channel"
	depends on FSL_DMA
	default y
	---help---
	Enable the self test for each DMA channel. A self test will be
	performed after the channel probed to ensure the DMA works well.

	config DMA_ENGINE		config DMA_ENGINE
	bool		bool

drivers/dma/dmaengine.c

+6 −9

Original line number	Original line	Diff line number	Diff line
	@@ -362,7 +362,6 @@ int dma_async_device_register(struct dma_device *device)

	BUG_ON(!device->device_alloc_chan_resources);		BUG_ON(!device->device_alloc_chan_resources);
	BUG_ON(!device->device_free_chan_resources);		BUG_ON(!device->device_free_chan_resources);
	BUG_ON(!device->device_dependency_added);
	BUG_ON(!device->device_is_tx_complete);		BUG_ON(!device->device_is_tx_complete);
	BUG_ON(!device->device_issue_pending);		BUG_ON(!device->device_issue_pending);
	BUG_ON(!device->dev);		BUG_ON(!device->dev);
	@@ -479,7 +478,8 @@ dma_async_memcpy_buf_to_buf(struct dma_chan chan, void dest,

	dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);		dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
	dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);		dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
	tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0);		tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
			DMA_CTRL_ACK);

	if (!tx) {		if (!tx) {
	dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);		dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);
	@@ -487,7 +487,6 @@ dma_async_memcpy_buf_to_buf(struct dma_chan chan, void dest,
	return -ENOMEM;		return -ENOMEM;
	}		}

	tx->ack = 1;
	tx->callback = NULL;		tx->callback = NULL;
	cookie = tx->tx_submit(tx);		cookie = tx->tx_submit(tx);

	@@ -525,7 +524,8 @@ dma_async_memcpy_buf_to_pg(struct dma_chan chan, struct page page,

	dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);		dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
	dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);		dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
	tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0);		tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
			DMA_CTRL_ACK);

	if (!tx) {		if (!tx) {
	dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);		dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);
	@@ -533,7 +533,6 @@ dma_async_memcpy_buf_to_pg(struct dma_chan chan, struct page page,
	return -ENOMEM;		return -ENOMEM;
	}		}

	tx->ack = 1;
	tx->callback = NULL;		tx->callback = NULL;
	cookie = tx->tx_submit(tx);		cookie = tx->tx_submit(tx);

	@@ -574,7 +573,8 @@ dma_async_memcpy_pg_to_pg(struct dma_chan chan, struct page dest_pg,
	dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);		dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
	dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len,		dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len,
	DMA_FROM_DEVICE);		DMA_FROM_DEVICE);
	tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0);		tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
			DMA_CTRL_ACK);

	if (!tx) {		if (!tx) {
	dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE);		dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE);
	@@ -582,7 +582,6 @@ dma_async_memcpy_pg_to_pg(struct dma_chan chan, struct page dest_pg,
	return -ENOMEM;		return -ENOMEM;
	}		}

	tx->ack = 1;
	tx->callback = NULL;		tx->callback = NULL;
	cookie = tx->tx_submit(tx);		cookie = tx->tx_submit(tx);

	@@ -600,8 +599,6 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
	{		{
	tx->chan = chan;		tx->chan = chan;
	spin_lock_init(&tx->lock);		spin_lock_init(&tx->lock);
	INIT_LIST_HEAD(&tx->depend_node);
	INIT_LIST_HEAD(&tx->depend_list);
	}		}
	EXPORT_SYMBOL(dma_async_tx_descriptor_init);		EXPORT_SYMBOL(dma_async_tx_descriptor_init);

Admin message