Commit a08abd8c authored by Dan Williams's avatar Dan Williams
Browse files

async_tx: structify submission arguments, add scribble 



Prepare the api for the arrival of a new parameter, 'scribble'.  This
will allow callers to identify scratchpad memory for dma address or page
address conversions.  As this adds yet another parameter, take this
opportunity to convert the common submission parameters (flags,
dependency, callback, and callback argument) into an object that is
passed by reference.

Also, take this opportunity to fix up the kerneldoc and add notes about
the relevant ASYNC_TX_* flags for each routine.

[ Impact: moves api pass-by-value parameters to a pass-by-reference struct ]

Signed-off-by: default avatarAndre Noll <maan@systemlinux.org>
Acked-by: default avatarMaciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: default avatarDan Williams <dan.j.williams@intel.com>
parent 88ba2aa5

Documentation/crypto/async-tx-api.txt

+1 −5
Original line number Diff line number Diff line
@@ -54,11 +54,7 @@ features surfaced as a result: 


3.1 General format of the API:

struct dma_async_tx_descriptor *

async_<operation>(<op specific parameters>,

		  enum async_tx_flags flags,

        	  struct dma_async_tx_descriptor *dependency,

        	  dma_async_tx_callback callback_routine,

		  void *callback_parameter);

async_<operation>(<op specific parameters>, struct async_submit ctl *submit)



3.2 Supported operations:

memcpy  - memory copy between a source and a destination buffer

crypto/async_tx/async_memcpy.c

+13 −13
Original line number Diff line number Diff line
@@ -33,28 +33,28 @@ 
 * async_memcpy - attempt to copy memory with a dma engine.

 * @dest: destination page

 * @src: src page

 * @offset: offset in pages to start transaction

 * @dest_offset: offset into 'dest' to start transaction

 * @src_offset: offset into 'src' to start transaction

 * @len: length in bytes

 * @flags: ASYNC_TX_ACK

 * @depend_tx: memcpy depends on the result of this transaction

 * @cb_fn: function to call when the memcpy completes

 * @cb_param: parameter to pass to the callback routine

 * @submit: submission / completion modifiers

 *

 * honored flags: ASYNC_TX_ACK

 */

struct dma_async_tx_descriptor *

async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,

	unsigned int src_offset, size_t len, enum async_tx_flags flags,

	struct dma_async_tx_descriptor *depend_tx,

	dma_async_tx_callback cb_fn, void *cb_param)

	     unsigned int src_offset, size_t len,

	     struct async_submit_ctl *submit)

{

	struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMCPY,

	struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY,

						      &dest, 1, &src, 1, len);

	struct dma_device *device = chan ? chan->device : NULL;

	struct dma_async_tx_descriptor *tx = NULL;



	if (device) {

		dma_addr_t dma_dest, dma_src;

		unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;

		unsigned long dma_prep_flags;



		dma_prep_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;

		dma_dest = dma_map_page(device->dev, dest, dest_offset, len,

					DMA_FROM_DEVICE);
@@ -67,13 +67,13 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, 


	if (tx) {

		pr_debug("%s: (async) len: %zu\n", __func__, len);

		async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);

		async_tx_submit(chan, tx, submit);

	} else {

		void *dest_buf, *src_buf;

		pr_debug("%s: (sync) len: %zu\n", __func__, len);



		/* wait for any prerequisite operations */

		async_tx_quiesce(&depend_tx);

		async_tx_quiesce(&submit->depend_tx);



		dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset;

		src_buf = kmap_atomic(src, KM_USER1) + src_offset;
@@ -83,7 +83,7 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, 
		kunmap_atomic(dest_buf, KM_USER0);

		kunmap_atomic(src_buf, KM_USER1);



		async_tx_sync_epilog(cb_fn, cb_param);

		async_tx_sync_epilog(submit);

	}



	return tx;

crypto/async_tx/async_memset.c

+11 −14
Original line number Diff line number Diff line
@@ -35,26 +35,23 @@ 
 * @val: fill value

 * @offset: offset in pages to start transaction

 * @len: length in bytes

 * @flags: ASYNC_TX_ACK

 * @depend_tx: memset depends on the result of this transaction

 * @cb_fn: function to call when the memcpy completes

 * @cb_param: parameter to pass to the callback routine

 *

 * honored flags: ASYNC_TX_ACK

 */

struct dma_async_tx_descriptor *

async_memset(struct page *dest, int val, unsigned int offset,

	size_t len, enum async_tx_flags flags,

	struct dma_async_tx_descriptor *depend_tx,

	dma_async_tx_callback cb_fn, void *cb_param)

async_memset(struct page *dest, int val, unsigned int offset, size_t len,

	     struct async_submit_ctl *submit)

{

	struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMSET,

	struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMSET,

						      &dest, 1, NULL, 0, len);

	struct dma_device *device = chan ? chan->device : NULL;

	struct dma_async_tx_descriptor *tx = NULL;



	if (device) {

		dma_addr_t dma_dest;

		unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;

		unsigned long dma_prep_flags;



		dma_prep_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;

		dma_dest = dma_map_page(device->dev, dest, offset, len,

					DMA_FROM_DEVICE);
@@ -64,19 +61,19 @@ async_memset(struct page *dest, int val, unsigned int offset, 


	if (tx) {

		pr_debug("%s: (async) len: %zu\n", __func__, len);

		async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);

		async_tx_submit(chan, tx, submit);

	} else { /* run the memset synchronously */

		void *dest_buf;

		pr_debug("%s: (sync) len: %zu\n", __func__, len);



		dest_buf = (void *) (((char *) page_address(dest)) + offset);

		dest_buf = page_address(dest) + offset;



		/* wait for any prerequisite operations */

		async_tx_quiesce(&depend_tx);

		async_tx_quiesce(&submit->depend_tx);



		memset(dest_buf, val, len);



		async_tx_sync_epilog(cb_fn, cb_param);

		async_tx_sync_epilog(submit);

	}



	return tx;

crypto/async_tx/async_tx.c

+27 −24
Original line number Diff line number Diff line
@@ -45,13 +45,15 @@ static void __exit async_tx_exit(void) 
/**

 * __async_tx_find_channel - find a channel to carry out the operation or let

 *	the transaction execute synchronously

 * @depend_tx: transaction dependency

 * @submit: transaction dependency and submission modifiers

 * @tx_type: transaction type

 */

struct dma_chan *

__async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,

__async_tx_find_channel(struct async_submit_ctl *submit,

			enum dma_transaction_type tx_type)

{

	struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;



	/* see if we can keep the chain on one channel */

	if (depend_tx &&

	    dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
@@ -144,13 +146,14 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, 




/**

 * submit_disposition - while holding depend_tx->lock we must avoid submitting

 * 	new operations to prevent a circular locking dependency with

 * 	drivers that already hold a channel lock when calling

 * 	async_tx_run_dependencies.

 * submit_disposition - flags for routing an incoming operation

 * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock

 * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch

 * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly

 *

 * while holding depend_tx->lock we must avoid submitting new operations

 * to prevent a circular locking dependency with drivers that already

 * hold a channel lock when calling async_tx_run_dependencies.

 */

enum submit_disposition {

	ASYNC_TX_SUBMITTED,
@@ -160,11 +163,12 @@ enum submit_disposition { 


void

async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,

	enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,

	dma_async_tx_callback cb_fn, void *cb_param)

		struct async_submit_ctl *submit)

{

	tx->callback = cb_fn;

	tx->callback_param = cb_param;

	struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;



	tx->callback = submit->cb_fn;

	tx->callback_param = submit->cb_param;



	if (depend_tx) {

		enum submit_disposition s;
@@ -220,7 +224,7 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, 
		tx->tx_submit(tx);

	}



	if (flags & ASYNC_TX_ACK)

	if (submit->flags & ASYNC_TX_ACK)

		async_tx_ack(tx);



	if (depend_tx)
@@ -229,21 +233,20 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, 
EXPORT_SYMBOL_GPL(async_tx_submit);



/**

 * async_trigger_callback - schedules the callback function to be run after

 * any dependent operations have been completed.

 * @flags: ASYNC_TX_ACK

 * @depend_tx: 'callback' requires the completion of this transaction

 * @cb_fn: function to call after depend_tx completes

 * @cb_param: parameter to pass to the callback routine

 * async_trigger_callback - schedules the callback function to be run

 * @submit: submission and completion parameters

 *

 * honored flags: ASYNC_TX_ACK

 *

 * The callback is run after any dependent operations have completed.

 */

struct dma_async_tx_descriptor *

async_trigger_callback(enum async_tx_flags flags,

	struct dma_async_tx_descriptor *depend_tx,

	dma_async_tx_callback cb_fn, void *cb_param)

async_trigger_callback(struct async_submit_ctl *submit)

{

	struct dma_chan *chan;

	struct dma_device *device;

	struct dma_async_tx_descriptor *tx;

	struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;



	if (depend_tx) {

		chan = depend_tx->chan;
@@ -262,14 +265,14 @@ async_trigger_callback(enum async_tx_flags flags, 
	if (tx) {

		pr_debug("%s: (async)\n", __func__);



		async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);

		async_tx_submit(chan, tx, submit);

	} else {

		pr_debug("%s: (sync)\n", __func__);



		/* wait for any prerequisite operations */

		async_tx_quiesce(&depend_tx);

		async_tx_quiesce(&submit->depend_tx);



		async_tx_sync_epilog(cb_fn, cb_param);

		async_tx_sync_epilog(submit);

	}



	return tx;

crypto/async_tx/async_xor.c

+60 −63
Original line number Diff line number Diff line
@@ -34,18 +34,16 @@ 
static __async_inline struct dma_async_tx_descriptor *

do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,

	     unsigned int offset, int src_cnt, size_t len,

	     enum async_tx_flags flags,

	     struct dma_async_tx_descriptor *depend_tx,

	     dma_async_tx_callback cb_fn, void *cb_param)

	     struct async_submit_ctl *submit)

{

	struct dma_device *dma = chan->device;

	dma_addr_t *dma_src = (dma_addr_t *) src_list;

	struct dma_async_tx_descriptor *tx = NULL;

	int src_off = 0;

	int i;

	dma_async_tx_callback _cb_fn;

	void *_cb_param;

	enum async_tx_flags async_flags;

	dma_async_tx_callback cb_fn_orig = submit->cb_fn;

	void *cb_param_orig = submit->cb_param;

	enum async_tx_flags flags_orig = submit->flags;

	enum dma_ctrl_flags dma_flags;

	int xor_src_cnt;

	dma_addr_t dma_dest;
@@ -63,7 +61,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 
	}



	while (src_cnt) {

		async_flags = flags;

		submit->flags = flags_orig;

		dma_flags = 0;

		xor_src_cnt = min(src_cnt, dma->max_xor);

		/* if we are submitting additional xors, leave the chain open,
@@ -71,15 +69,15 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 
		 * buffer mapped

		 */

		if (src_cnt > xor_src_cnt) {

			async_flags &= ~ASYNC_TX_ACK;

			submit->flags &= ~ASYNC_TX_ACK;

			dma_flags = DMA_COMPL_SKIP_DEST_UNMAP;

			_cb_fn = NULL;

			_cb_param = NULL;

			submit->cb_fn = NULL;

			submit->cb_param = NULL;

		} else {

			_cb_fn = cb_fn;

			_cb_param = cb_param;

			submit->cb_fn = cb_fn_orig;

			submit->cb_param = cb_param_orig;

		}

		if (_cb_fn)

		if (submit->cb_fn)

			dma_flags |= DMA_PREP_INTERRUPT;



		/* Since we have clobbered the src_list we are committed
@@ -90,7 +88,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 
					      xor_src_cnt, len, dma_flags);



		if (unlikely(!tx))

			async_tx_quiesce(&depend_tx);

			async_tx_quiesce(&submit->depend_tx);



		/* spin wait for the preceeding transactions to complete */

		while (unlikely(!tx)) {
@@ -101,10 +99,8 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 
						      dma_flags);

		}



		async_tx_submit(chan, tx, async_flags, depend_tx, _cb_fn,

				_cb_param);



		depend_tx = tx;

		async_tx_submit(chan, tx, submit);

		submit->depend_tx = tx;



		if (src_cnt > xor_src_cnt) {

			/* drop completed sources */
@@ -123,8 +119,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 


static void

do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,

	    int src_cnt, size_t len, enum async_tx_flags flags,

	    dma_async_tx_callback cb_fn, void *cb_param)

	    int src_cnt, size_t len, struct async_submit_ctl *submit)

{

	int i;

	int xor_src_cnt;
@@ -139,7 +134,7 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, 
	/* set destination address */

	dest_buf = page_address(dest) + offset;



	if (flags & ASYNC_TX_XOR_ZERO_DST)

	if (submit->flags & ASYNC_TX_XOR_ZERO_DST)

		memset(dest_buf, 0, len);



	while (src_cnt > 0) {
@@ -152,33 +147,35 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, 
		src_off += xor_src_cnt;

	}



	async_tx_sync_epilog(cb_fn, cb_param);

	async_tx_sync_epilog(submit);

}



/**

 * async_xor - attempt to xor a set of blocks with a dma engine.

 *	xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST

 *	flag must be set to not include dest data in the calculation.  The

 *	assumption with dma eninges is that they only use the destination

 *	buffer as a source when it is explicity specified in the source list.

 * @dest: destination page

 * @src_list: array of source pages (if the dest is also a source it must be

 *	at index zero).  The contents of this array may be overwritten.

 * @offset: offset in pages to start transaction

 * @src_list: array of source pages

 * @offset: common src/dst offset to start transaction

 * @src_cnt: number of source pages

 * @len: length in bytes

 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST, ASYNC_TX_ACK

 * @depend_tx: xor depends on the result of this transaction.

 * @cb_fn: function to call when the xor completes

 * @cb_param: parameter to pass to the callback routine

 * @submit: submission / completion modifiers

 *

 * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST

 *

 * xor_blocks always uses the dest as a source so the

 * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in

 * the calculation.  The assumption with dma eninges is that they only

 * use the destination buffer as a source when it is explicity specified

 * in the source list.

 *

 * src_list note: if the dest is also a source it must be at index zero.

 * The contents of this array will be overwritten if a scribble region

 * is not specified.

 */

struct dma_async_tx_descriptor *

async_xor(struct page *dest, struct page **src_list, unsigned int offset,

	int src_cnt, size_t len, enum async_tx_flags flags,

	struct dma_async_tx_descriptor *depend_tx,

	dma_async_tx_callback cb_fn, void *cb_param)

	  int src_cnt, size_t len, struct async_submit_ctl *submit)

{

	struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR,

	struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,

						      &dest, 1, src_list,

						      src_cnt, len);

	BUG_ON(src_cnt <= 1);
@@ -188,7 +185,7 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, 
		pr_debug("%s (async): len: %zu\n", __func__, len);



		return do_async_xor(chan, dest, src_list, offset, src_cnt, len,

				    flags, depend_tx, cb_fn, cb_param);

				    submit);

	} else {

		/* run the xor synchronously */

		pr_debug("%s (sync): len: %zu\n", __func__, len);
@@ -196,16 +193,15 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, 
		/* in the sync case the dest is an implied source

		 * (assumes the dest is the first source)

		 */

		if (flags & ASYNC_TX_XOR_DROP_DST) {

		if (submit->flags & ASYNC_TX_XOR_DROP_DST) {

			src_cnt--;

			src_list++;

		}



		/* wait for any prerequisite operations */

		async_tx_quiesce(&depend_tx);

		async_tx_quiesce(&submit->depend_tx);



		do_sync_xor(dest, src_list, offset, src_cnt, len,

			    flags, cb_fn, cb_param);

		do_sync_xor(dest, src_list, offset, src_cnt, len, submit);



		return NULL;

	}
@@ -222,25 +218,25 @@ static int page_is_zero(struct page *p, unsigned int offset, size_t len) 
/**

 * async_xor_val - attempt a xor parity check with a dma engine.

 * @dest: destination page used if the xor is performed synchronously

 * @src_list: array of source pages.  The dest page must be listed as a source

 * 	at index zero.  The contents of this array may be overwritten.

 * @src_list: array of source pages

 * @offset: offset in pages to start transaction

 * @src_cnt: number of source pages

 * @len: length in bytes

 * @result: 0 if sum == 0 else non-zero

 * @flags: ASYNC_TX_ACK

 * @depend_tx: xor depends on the result of this transaction.

 * @cb_fn: function to call when the xor completes

 * @cb_param: parameter to pass to the callback routine

 * @submit: submission / completion modifiers

 *

 * honored flags: ASYNC_TX_ACK

 *

 * src_list note: if the dest is also a source it must be at index zero.

 * The contents of this array will be overwritten if a scribble region

 * is not specified.

 */

struct dma_async_tx_descriptor *

async_xor_val(struct page *dest, struct page **src_list,

	unsigned int offset, int src_cnt, size_t len,

	u32 *result, enum async_tx_flags flags,

	struct dma_async_tx_descriptor *depend_tx,

	dma_async_tx_callback cb_fn, void *cb_param)

async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,

	      int src_cnt, size_t len, u32 *result,

	      struct async_submit_ctl *submit)

{

	struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR_VAL,

	struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR_VAL,

						      &dest, 1, src_list,

						      src_cnt, len);

	struct dma_device *device = chan ? chan->device : NULL;
@@ -250,11 +246,12 @@ async_xor_val(struct page *dest, struct page **src_list, 


	if (device && src_cnt <= device->max_xor) {

		dma_addr_t *dma_src = (dma_addr_t *) src_list;

		unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;

		unsigned long dma_prep_flags;

		int i;



		pr_debug("%s: (async) len: %zu\n", __func__, len);



		dma_prep_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;

		for (i = 0; i < src_cnt; i++)

			dma_src[i] = dma_map_page(device->dev, src_list[i],

						  offset, len, DMA_TO_DEVICE);
@@ -263,7 +260,7 @@ async_xor_val(struct page *dest, struct page **src_list, 
						     len, result,

						     dma_prep_flags);

		if (unlikely(!tx)) {

			async_tx_quiesce(&depend_tx);

			async_tx_quiesce(&submit->depend_tx);



			while (!tx) {

				dma_async_issue_pending(chan);
@@ -273,23 +270,23 @@ async_xor_val(struct page *dest, struct page **src_list, 
			}

		}



		async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);

		async_tx_submit(chan, tx, submit);

	} else {

		unsigned long xor_flags = flags;

		enum async_tx_flags flags_orig = submit->flags;



		pr_debug("%s: (sync) len: %zu\n", __func__, len);



		xor_flags |= ASYNC_TX_XOR_DROP_DST;

		xor_flags &= ~ASYNC_TX_ACK;

		submit->flags |= ASYNC_TX_XOR_DROP_DST;

		submit->flags &= ~ASYNC_TX_ACK;



		tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags,

			depend_tx, NULL, NULL);

		tx = async_xor(dest, src_list, offset, src_cnt, len, submit);



		async_tx_quiesce(&tx);



		*result = page_is_zero(dest, offset, len) ? 0 : 1;



		async_tx_sync_epilog(cb_fn, cb_param);

		async_tx_sync_epilog(submit);

		submit->flags = flags_orig;

	}



	return tx;

CRA Git | Maintained and supported by SUSTech CRA and CCSE