Commit 5d22d47b authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'sfc-filter-locking' 



Edward Cree says:

====================
sfc: rework locking around filter management

The use of a spinlock to protect filter state combined with the need for a
 sleeping operation (MCDI) to apply that state to the NIC (on EF10) led to
 unfixable race conditions, around the handling of filter restoration after
 an MC reboot.
So, this patch series removes the requirement to be able to modify the SW
 filter table from atomic context, by using a workqueue to request
 asynchronous filter operations (which are needed for ARFS).  Then, the
 filter table locks are changed to mutexes, replacing the dance of spinlocks
 and 'busy' flags.  Also, a mutex is added to protect the RSS context state,
 since otherwise a similar race is possible around restoring that after an
 MC reboot.  While we're at it, fix a couple of other related bugs.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents c709002c a8e8fbeb

drivers/net/ethernet/sfc/ef10.c

+190 −355
Original line number Diff line number Diff line
@@ -96,17 +96,15 @@ struct efx_ef10_filter_table { 
		MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MAXNUM * 2];

	unsigned int rx_match_count;



	struct rw_semaphore lock; /* Protects entries */

	struct {

		unsigned long spec;	/* pointer to spec plus flag bits */

/* BUSY flag indicates that an update is in progress.  AUTO_OLD is

 * used to mark and sweep MAC filters for the device address lists.

 */

#define EFX_EF10_FILTER_FLAG_BUSY	1UL

/* AUTO_OLD is used to mark and sweep MAC filters for the device address lists. */

/* unused flag	1UL */

#define EFX_EF10_FILTER_FLAG_AUTO_OLD	2UL

#define EFX_EF10_FILTER_FLAGS		3UL

		u64 handle;		/* firmware handle */

	} *entry;

	wait_queue_head_t waitq;

/* Shadow of net_device address lists, guarded by mac_lock */

	struct efx_ef10_dev_addr dev_uc_list[EFX_EF10_FILTER_DEV_UC_MAX];

	struct efx_ef10_dev_addr dev_mc_list[EFX_EF10_FILTER_DEV_MC_MAX];
@@ -1501,6 +1499,7 @@ static void efx_ef10_reset_mc_allocations(struct efx_nic *efx) 


	/* All our allocations have been reset */

	nic_data->must_realloc_vis = true;

	nic_data->must_restore_rss_contexts = true;

	nic_data->must_restore_filters = true;

	nic_data->must_restore_piobufs = true;

	efx_ef10_forget_old_piobufs(efx);
@@ -2901,6 +2900,8 @@ static int efx_ef10_rx_push_rss_context_config(struct efx_nic *efx, 
{

	int rc;



	WARN_ON(!mutex_is_locked(&efx->rss_lock));



	if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) {

		rc = efx_ef10_alloc_rss_context(efx, true, ctx, NULL);

		if (rc)
@@ -2931,6 +2932,8 @@ static int efx_ef10_rx_pull_rss_context_config(struct efx_nic *efx, 
	size_t outlen;

	int rc, i;



	WARN_ON(!mutex_is_locked(&efx->rss_lock));



	BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN !=

		     MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN);
@@ -2974,14 +2977,25 @@ static int efx_ef10_rx_pull_rss_context_config(struct efx_nic *efx, 


static int efx_ef10_rx_pull_rss_config(struct efx_nic *efx)

{

	return efx_ef10_rx_pull_rss_context_config(efx, &efx->rss_context);

	int rc;



	mutex_lock(&efx->rss_lock);

	rc = efx_ef10_rx_pull_rss_context_config(efx, &efx->rss_context);

	mutex_unlock(&efx->rss_lock);

	return rc;

}



static void efx_ef10_rx_restore_rss_contexts(struct efx_nic *efx)

{

	struct efx_ef10_nic_data *nic_data = efx->nic_data;

	struct efx_rss_context *ctx;

	int rc;



	WARN_ON(!mutex_is_locked(&efx->rss_lock));



	if (!nic_data->must_restore_rss_contexts)

		return;



	list_for_each_entry(ctx, &efx->rss_context.list, list) {

		/* previous NIC RSS context is gone */

		ctx->context_id = EFX_EF10_RSS_CONTEXT_INVALID;
@@ -2995,6 +3009,7 @@ static void efx_ef10_rx_restore_rss_contexts(struct efx_nic *efx) 
				   "; RSS filters may fail to be applied\n",

				   ctx->user_id, rc);

	}

	nic_data->must_restore_rss_contexts = false;

}



static int efx_ef10_pf_rx_push_rss_config(struct efx_nic *efx, bool user,
@@ -4302,26 +4317,35 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, 
				  struct efx_filter_spec *spec,

				  bool replace_equal)

{

	struct efx_ef10_filter_table *table = efx->filter_state;

	DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);

	struct efx_ef10_nic_data *nic_data = efx->nic_data;

	struct efx_ef10_filter_table *table;

	struct efx_filter_spec *saved_spec;

	struct efx_rss_context *ctx = NULL;

	unsigned int match_pri, hash;

	unsigned int priv_flags;

	bool rss_locked = false;

	bool replacing = false;

	unsigned int depth, i;

	int ins_index = -1;

	DEFINE_WAIT(wait);

	bool is_mc_recip;

	s32 rc;



	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_write(&table->lock);



	/* For now, only support RX filters */

	if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=

	    EFX_FILTER_FLAG_RX)

		return -EINVAL;

	    EFX_FILTER_FLAG_RX) {

		rc = -EINVAL;

		goto out_unlock;

	}



	rc = efx_ef10_filter_pri(table, spec);

	if (rc < 0)

		return rc;

		goto out_unlock;

	match_pri = rc;



	hash = efx_ef10_filter_hash(spec);
@@ -4330,28 +4354,26 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, 
		bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);



	if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {

		mutex_lock(&efx->rss_lock);

		rss_locked = true;

		if (spec->rss_context)

			ctx = efx_find_rss_context_entry(spec->rss_context,

							 &efx->rss_context.list);

			ctx = efx_find_rss_context_entry(efx, spec->rss_context);

		else

			ctx = &efx->rss_context;

		if (!ctx)

			return -ENOENT;

		if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID)

			return -EOPNOTSUPP;

		if (!ctx) {

			rc = -ENOENT;

			goto out_unlock;

		}

		if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) {

			rc = -EOPNOTSUPP;

			goto out_unlock;

		}

	}



	/* Find any existing filters with the same match tuple or

	 * else a free slot to insert at.  If any of them are busy,

	 * we have to wait and retry.

	 * else a free slot to insert at.

	 */

	for (;;) {

		unsigned int depth = 1;

		unsigned int i;



		spin_lock_bh(&efx->filter_lock);



		for (;;) {

	for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {

		i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);

		saved_spec = efx_ef10_filter_entry_spec(table, i);
@@ -4359,9 +4381,6 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, 
			if (ins_index < 0)

				ins_index = i;

		} else if (efx_ef10_filter_equal(spec, saved_spec)) {

				if (table->entry[i].spec &

				    EFX_EF10_FILTER_FLAG_BUSY)

					break;

			if (spec->priority < saved_spec->priority &&

			    spec->priority != EFX_FILTER_PRI_AUTO) {

				rc = -EPERM;
@@ -4376,7 +4395,7 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, 
					goto out_unlock;

				}

				ins_index = i;

					goto found;

				break;

			} else if (spec->priority >

				   saved_spec->priority ||

				   (spec->priority ==
@@ -4388,33 +4407,17 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, 
					__set_bit(depth, mc_rem_map);

			}

		}

	}



			/* Once we reach the maximum search depth, use

			 * the first suitable slot or return -EBUSY if

			 * there was none

	/* Once we reach the maximum search depth, use the first suitable

	 * slot, or return -EBUSY if there was none

	 */

			if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) {

	if (ins_index < 0) {

		rc = -EBUSY;

		goto out_unlock;

	}

				goto found;

			}



			++depth;

		}



		prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE);

		spin_unlock_bh(&efx->filter_lock);

		schedule();

	}



found:

	/* Create a software table entry if necessary, and mark it

	 * busy.  We might yet fail to insert, but any attempt to

	 * insert a conflicting filter while we're waiting for the

	 * firmware must find the busy entry.

	 */

	/* Create a software table entry if necessary. */

	saved_spec = efx_ef10_filter_entry_spec(table, ins_index);

	if (saved_spec) {

		if (spec->priority == EFX_FILTER_PRI_AUTO &&
@@ -4438,28 +4441,19 @@ found: 
		*saved_spec = *spec;

		priv_flags = 0;

	}

	efx_ef10_filter_set_entry(table, ins_index, saved_spec,

				  priv_flags | EFX_EF10_FILTER_FLAG_BUSY);



	/* Mark lower-priority multicast recipients busy prior to removal */

	if (is_mc_recip) {

		unsigned int depth, i;



		for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {

			i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);

			if (test_bit(depth, mc_rem_map))

				table->entry[i].spec |=

					EFX_EF10_FILTER_FLAG_BUSY;

		}

	}



	spin_unlock_bh(&efx->filter_lock);

	efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags);



	/* Actually insert the filter on the HW */

	rc = efx_ef10_filter_push(efx, spec, &table->entry[ins_index].handle,

				  ctx, replacing);



	if (rc == -EINVAL && nic_data->must_realloc_vis)

		/* The MC rebooted under us, causing it to reject our filter

		 * insertion as pointing to an invalid VI (spec->dmaq_id).

		 */

		rc = -EAGAIN;



	/* Finalise the software table entry */

	spin_lock_bh(&efx->filter_lock);

	if (rc == 0) {

		if (replacing) {

			/* Update the fields that may differ */
@@ -4475,6 +4469,12 @@ found: 
	} else if (!replacing) {

		kfree(saved_spec);

		saved_spec = NULL;

	} else {

		/* We failed to replace, so the old filter is still present.

		 * Roll back the software table to reflect this.  In fact the

		 * efx_ef10_filter_set_entry() call below will do the right

		 * thing, so nothing extra is needed here.

		 */

	}

	efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags);
@@ -4496,7 +4496,6 @@ found: 
			priv_flags = efx_ef10_filter_entry_flags(table, i);



			if (rc == 0) {

				spin_unlock_bh(&efx->filter_lock);

				MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,

					       MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);

				MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
@@ -4504,15 +4503,12 @@ found: 
				rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,

						  inbuf, sizeof(inbuf),

						  NULL, 0, NULL);

				spin_lock_bh(&efx->filter_lock);

			}



			if (rc == 0) {

				kfree(saved_spec);

				saved_spec = NULL;

				priv_flags = 0;

			} else {

				priv_flags &= ~EFX_EF10_FILTER_FLAG_BUSY;

			}

			efx_ef10_filter_set_entry(table, i, saved_spec,

						  priv_flags);
@@ -4523,10 +4519,11 @@ found: 
	if (rc == 0)

		rc = efx_ef10_make_filter_id(match_pri, ins_index);



	wake_up_all(&table->waitq);

out_unlock:

	spin_unlock_bh(&efx->filter_lock);

	finish_wait(&table->waitq, &wait);

	if (rss_locked)

		mutex_unlock(&efx->rss_lock);

	up_write(&table->lock);

	up_read(&efx->filter_sem);

	return rc;

}
@@ -4539,6 +4536,8 @@ static void efx_ef10_filter_update_rx_scatter(struct efx_nic *efx) 
 * If !by_index, remove by ID

 * If by_index, remove by index

 * Filter ID may come from userland and must be range-checked.

 * Caller must hold efx->filter_sem for read, and efx->filter_state->lock

 * for write.

 */

static int efx_ef10_filter_remove_internal(struct efx_nic *efx,

					   unsigned int priority_mask,
@@ -4553,45 +4552,23 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, 
	DEFINE_WAIT(wait);

	int rc;



	/* Find the software table entry and mark it busy.  Don't

	 * remove it yet; any attempt to update while we're waiting

	 * for the firmware must find the busy entry.

	 */

	for (;;) {

		spin_lock_bh(&efx->filter_lock);

		if (!(table->entry[filter_idx].spec &

		      EFX_EF10_FILTER_FLAG_BUSY))

			break;

		prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE);

		spin_unlock_bh(&efx->filter_lock);

		schedule();

	}



	spec = efx_ef10_filter_entry_spec(table, filter_idx);

	if (!spec ||

	    (!by_index &&

	     efx_ef10_filter_pri(table, spec) !=

	     efx_ef10_filter_get_unsafe_pri(filter_id))) {

		rc = -ENOENT;

		goto out_unlock;

	}

	     efx_ef10_filter_get_unsafe_pri(filter_id)))

		return -ENOENT;



	if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO &&

	    priority_mask == (1U << EFX_FILTER_PRI_AUTO)) {

		/* Just remove flags */

		spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO;

		table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD;

		rc = 0;

		goto out_unlock;

	}



	if (!(priority_mask & (1U << spec->priority))) {

		rc = -ENOENT;

		goto out_unlock;

		return 0;

	}



	table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;

	spin_unlock_bh(&efx->filter_lock);

	if (!(priority_mask & (1U << spec->priority)))

		return -ENOENT;



	if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {

		/* Reset to an automatic filter */
@@ -4609,7 +4586,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, 
					  &efx->rss_context,

					  true);



		spin_lock_bh(&efx->filter_lock);

		if (rc == 0)

			*spec = new_spec;

	} else {
@@ -4624,7 +4600,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, 
		rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP,

					inbuf, sizeof(inbuf), NULL, 0, NULL);



		spin_lock_bh(&efx->filter_lock);

		if ((rc == 0) || (rc == -ENOENT)) {

			/* Filter removed OK or didn't actually exist */

			kfree(spec);
@@ -4636,11 +4611,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, 
		}

	}



	table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY;

	wake_up_all(&table->waitq);

out_unlock:

	spin_unlock_bh(&efx->filter_lock);

	finish_wait(&table->waitq, &wait);

	return rc;

}
@@ -4648,17 +4618,33 @@ static int efx_ef10_filter_remove_safe(struct efx_nic *efx, 
				       enum efx_filter_priority priority,

				       u32 filter_id)

{

	return efx_ef10_filter_remove_internal(efx, 1U << priority,

					       filter_id, false);

	struct efx_ef10_filter_table *table;

	int rc;



	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_write(&table->lock);

	rc = efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id,

					     false);

	up_write(&table->lock);

	up_read(&efx->filter_sem);

	return rc;

}



/* Caller must hold efx->filter_sem for read */

static void efx_ef10_filter_remove_unsafe(struct efx_nic *efx,

					  enum efx_filter_priority priority,

					  u32 filter_id)

{

	struct efx_ef10_filter_table *table = efx->filter_state;



	if (filter_id == EFX_EF10_FILTER_ID_INVALID)

		return;

	efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, true);



	down_write(&table->lock);

	efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id,

					true);

	up_write(&table->lock);

}



static int efx_ef10_filter_get_safe(struct efx_nic *efx,
@@ -4666,11 +4652,13 @@ static int efx_ef10_filter_get_safe(struct efx_nic *efx, 
				    u32 filter_id, struct efx_filter_spec *spec)

{

	unsigned int filter_idx = efx_ef10_filter_get_unsafe_id(filter_id);

	struct efx_ef10_filter_table *table = efx->filter_state;

	const struct efx_filter_spec *saved_spec;

	struct efx_ef10_filter_table *table;

	int rc;



	spin_lock_bh(&efx->filter_lock);

	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_read(&table->lock);

	saved_spec = efx_ef10_filter_entry_spec(table, filter_idx);

	if (saved_spec && saved_spec->priority == priority &&

	    efx_ef10_filter_pri(table, saved_spec) ==
@@ -4680,13 +4668,15 @@ static int efx_ef10_filter_get_safe(struct efx_nic *efx, 
	} else {

		rc = -ENOENT;

	}

	spin_unlock_bh(&efx->filter_lock);

	up_read(&table->lock);

	up_read(&efx->filter_sem);

	return rc;

}



static int efx_ef10_filter_clear_rx(struct efx_nic *efx,

				    enum efx_filter_priority priority)

{

	struct efx_ef10_filter_table *table;

	unsigned int priority_mask;

	unsigned int i;

	int rc;
@@ -4694,31 +4684,40 @@ static int efx_ef10_filter_clear_rx(struct efx_nic *efx, 
	priority_mask = (((1U << (priority + 1)) - 1) &

			 ~(1U << EFX_FILTER_PRI_AUTO));



	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_write(&table->lock);

	for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) {

		rc = efx_ef10_filter_remove_internal(efx, priority_mask,

						     i, true);

		if (rc && rc != -ENOENT)

			return rc;

			break;

		rc = 0;

	}



	return 0;

	up_write(&table->lock);

	up_read(&efx->filter_sem);

	return rc;

}



static u32 efx_ef10_filter_count_rx_used(struct efx_nic *efx,

					 enum efx_filter_priority priority)

{

	struct efx_ef10_filter_table *table = efx->filter_state;

	struct efx_ef10_filter_table *table;

	unsigned int filter_idx;

	s32 count = 0;



	spin_lock_bh(&efx->filter_lock);

	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_read(&table->lock);

	for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {

		if (table->entry[filter_idx].spec &&

		    efx_ef10_filter_entry_spec(table, filter_idx)->priority ==

		    priority)

			++count;

	}

	spin_unlock_bh(&efx->filter_lock);

	up_read(&table->lock);

	up_read(&efx->filter_sem);

	return count;

}
@@ -4733,12 +4732,15 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx, 
				      enum efx_filter_priority priority,

				      u32 *buf, u32 size)

{

	struct efx_ef10_filter_table *table = efx->filter_state;

	struct efx_ef10_filter_table *table;

	struct efx_filter_spec *spec;

	unsigned int filter_idx;

	s32 count = 0;



	spin_lock_bh(&efx->filter_lock);

	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_read(&table->lock);



	for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {

		spec = efx_ef10_filter_entry_spec(table, filter_idx);

		if (spec && spec->priority == priority) {
@@ -4752,202 +4754,42 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx, 
					filter_idx);

		}

	}

	spin_unlock_bh(&efx->filter_lock);

	up_read(&table->lock);

	up_read(&efx->filter_sem);

	return count;

}



#ifdef CONFIG_RFS_ACCEL



static efx_mcdi_async_completer efx_ef10_filter_rfs_insert_complete;



static s32 efx_ef10_filter_rfs_insert(struct efx_nic *efx,

				      struct efx_filter_spec *spec)

{

	struct efx_ef10_filter_table *table = efx->filter_state;

	MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);

	struct efx_filter_spec *saved_spec;

	unsigned int hash, i, depth = 1;

	bool replacing = false;

	int ins_index = -1;

	u64 cookie;

	s32 rc;



	/* Must be an RX filter without RSS and not for a multicast

	 * destination address (RFS only works for connected sockets).

	 * These restrictions allow us to pass only a tiny amount of

	 * data through to the completion function.

	 */

	EFX_WARN_ON_PARANOID(spec->flags !=

			     (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_RX_SCATTER));

	EFX_WARN_ON_PARANOID(spec->priority != EFX_FILTER_PRI_HINT);

	EFX_WARN_ON_PARANOID(efx_filter_is_mc_recipient(spec));



	hash = efx_ef10_filter_hash(spec);



	spin_lock_bh(&efx->filter_lock);



	/* Find any existing filter with the same match tuple or else

	 * a free slot to insert at.  If an existing filter is busy,

	 * we have to give up.

	 */

	for (;;) {

		i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);

		saved_spec = efx_ef10_filter_entry_spec(table, i);



		if (!saved_spec) {

			if (ins_index < 0)

				ins_index = i;

		} else if (efx_ef10_filter_equal(spec, saved_spec)) {

			if (table->entry[i].spec & EFX_EF10_FILTER_FLAG_BUSY) {

				rc = -EBUSY;

				goto fail_unlock;

			}

			if (spec->priority < saved_spec->priority) {

				rc = -EPERM;

				goto fail_unlock;

			}

			ins_index = i;

			break;

		}



		/* Once we reach the maximum search depth, use the

		 * first suitable slot or return -EBUSY if there was

		 * none

		 */

		if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) {

			if (ins_index < 0) {

				rc = -EBUSY;

				goto fail_unlock;

			}

			break;

		}



		++depth;

	}



	/* Create a software table entry if necessary, and mark it

	 * busy.  We might yet fail to insert, but any attempt to

	 * insert a conflicting filter while we're waiting for the

	 * firmware must find the busy entry.

	 */

	saved_spec = efx_ef10_filter_entry_spec(table, ins_index);

	if (saved_spec) {

		replacing = true;

	} else {

		saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);

		if (!saved_spec) {

			rc = -ENOMEM;

			goto fail_unlock;

		}

		*saved_spec = *spec;

	}

	efx_ef10_filter_set_entry(table, ins_index, saved_spec,

				  EFX_EF10_FILTER_FLAG_BUSY);



	spin_unlock_bh(&efx->filter_lock);



	/* Pack up the variables needed on completion */

	cookie = replacing << 31 | ins_index << 16 | spec->dmaq_id;



	efx_ef10_filter_push_prep(efx, spec, inbuf,

				  table->entry[ins_index].handle, NULL,

				  replacing);

	efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),

			   MC_CMD_FILTER_OP_OUT_LEN,

			   efx_ef10_filter_rfs_insert_complete, cookie);



	return ins_index;



fail_unlock:

	spin_unlock_bh(&efx->filter_lock);

	return rc;

}



static void

efx_ef10_filter_rfs_insert_complete(struct efx_nic *efx, unsigned long cookie,

				    int rc, efx_dword_t *outbuf,

				    size_t outlen_actual)

{

	struct efx_ef10_filter_table *table = efx->filter_state;

	unsigned int ins_index, dmaq_id;

	struct efx_filter_spec *spec;

	bool replacing;



	/* Unpack the cookie */

	replacing = cookie >> 31;

	ins_index = (cookie >> 16) & (HUNT_FILTER_TBL_ROWS - 1);

	dmaq_id = cookie & 0xffff;



	spin_lock_bh(&efx->filter_lock);

	spec = efx_ef10_filter_entry_spec(table, ins_index);

	if (rc == 0) {

		table->entry[ins_index].handle =

			MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);

		if (replacing)

			spec->dmaq_id = dmaq_id;

	} else if (!replacing) {

		kfree(spec);

		spec = NULL;

	}

	efx_ef10_filter_set_entry(table, ins_index, spec, 0);

	spin_unlock_bh(&efx->filter_lock);



	wake_up_all(&table->waitq);

}



static void

efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx,

				    unsigned long filter_idx,

				    int rc, efx_dword_t *outbuf,

				    size_t outlen_actual);



static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,

					   unsigned int filter_idx)

{

	struct efx_ef10_filter_table *table = efx->filter_state;

	struct efx_filter_spec *spec =

		efx_ef10_filter_entry_spec(table, filter_idx);

	MCDI_DECLARE_BUF(inbuf,

			 MC_CMD_FILTER_OP_IN_HANDLE_OFST +

			 MC_CMD_FILTER_OP_IN_HANDLE_LEN);



	if (!spec ||

	    (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) ||

	    spec->priority != EFX_FILTER_PRI_HINT ||

	    !rps_may_expire_flow(efx->net_dev, spec->dmaq_id,

				 flow_id, filter_idx))

		return false;

	struct efx_ef10_filter_table *table;

	struct efx_filter_spec *spec;

	bool ret;



	MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,

		       MC_CMD_FILTER_OP_IN_OP_REMOVE);

	MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,

		       table->entry[filter_idx].handle);

	if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0,

			       efx_ef10_filter_rfs_expire_complete, filter_idx))

		return false;

	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_write(&table->lock);

	spec = efx_ef10_filter_entry_spec(table, filter_idx);



	table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;

	return true;

	if (!spec || spec->priority != EFX_FILTER_PRI_HINT) {

		ret = true;

		goto out_unlock;

	}



static void

efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx,

				    unsigned long filter_idx,

				    int rc, efx_dword_t *outbuf,

				    size_t outlen_actual)

{

	struct efx_ef10_filter_table *table = efx->filter_state;

	struct efx_filter_spec *spec =

		efx_ef10_filter_entry_spec(table, filter_idx);



	spin_lock_bh(&efx->filter_lock);

	if (rc == 0) {

		kfree(spec);

		efx_ef10_filter_set_entry(table, filter_idx, NULL, 0);

	if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id,

				 flow_id, filter_idx)) {

		ret = false;

		goto out_unlock;

	}

	table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY;

	wake_up_all(&table->waitq);

	spin_unlock_bh(&efx->filter_lock);



	ret = efx_ef10_filter_remove_internal(efx, 1U << spec->priority,

					      filter_idx, true) == 0;

out_unlock:

	up_write(&table->lock);

	up_read(&efx->filter_sem);

	return ret;

}



#endif /* CONFIG_RFS_ACCEL */
@@ -5142,9 +4984,9 @@ static int efx_ef10_filter_table_probe(struct efx_nic *efx) 
	table->vlan_filter =

		!!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);

	INIT_LIST_HEAD(&table->vlan_list);

	init_rwsem(&table->lock);



	efx->filter_state = table;

	init_waitqueue_head(&table->waitq);



	list_for_each_entry(vlan, &nic_data->vlan_list, list) {

		rc = efx_ef10_filter_add_vlan(efx, vlan->vid);
@@ -5186,7 +5028,8 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx) 
	if (!table)

		return;



	spin_lock_bh(&efx->filter_lock);

	down_write(&table->lock);

	mutex_lock(&efx->rss_lock);



	for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {

		spec = efx_ef10_filter_entry_spec(table, filter_idx);
@@ -5203,8 +5046,7 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx) 
			goto not_restored;

		}

		if (spec->rss_context)

			ctx = efx_find_rss_context_entry(spec->rss_context,

							 &efx->rss_context.list);

			ctx = efx_find_rss_context_entry(efx, spec->rss_context);

		else

			ctx = &efx->rss_context;

		if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
@@ -5224,15 +5066,11 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx) 
			}

		}



		table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;

		spin_unlock_bh(&efx->filter_lock);



		rc = efx_ef10_filter_push(efx, spec,

					  &table->entry[filter_idx].handle,

					  ctx, false);

		if (rc)

			failed++;

		spin_lock_bh(&efx->filter_lock);



		if (rc) {

not_restored:
@@ -5244,13 +5082,11 @@ not_restored: 


			kfree(spec);

			efx_ef10_filter_set_entry(table, filter_idx, NULL, 0);

		} else {

			table->entry[filter_idx].spec &=

				~EFX_EF10_FILTER_FLAG_BUSY;

		}

	}



	spin_unlock_bh(&efx->filter_lock);

	mutex_unlock(&efx->rss_lock);

	up_write(&table->lock);



	/* This can happen validly if the MC's capabilities have changed, so

	 * is not an error.
@@ -5318,6 +5154,8 @@ static void efx_ef10_filter_mark_one_old(struct efx_nic *efx, uint16_t *id) 
	struct efx_ef10_filter_table *table = efx->filter_state;

	unsigned int filter_idx;



	efx_rwsem_assert_write_locked(&table->lock);



	if (*id != EFX_EF10_FILTER_ID_INVALID) {

		filter_idx = efx_ef10_filter_get_unsafe_id(*id);

		if (!table->entry[filter_idx].spec)
@@ -5353,10 +5191,10 @@ static void efx_ef10_filter_mark_old(struct efx_nic *efx) 
	struct efx_ef10_filter_table *table = efx->filter_state;

	struct efx_ef10_filter_vlan *vlan;



	spin_lock_bh(&efx->filter_lock);

	down_write(&table->lock);

	list_for_each_entry(vlan, &table->vlan_list, list)

		_efx_ef10_filter_vlan_mark_old(efx, vlan);

	spin_unlock_bh(&efx->filter_lock);

	up_write(&table->lock);

}



static void efx_ef10_filter_uc_addr_list(struct efx_nic *efx)
@@ -5633,10 +5471,7 @@ static int efx_ef10_filter_insert_def(struct efx_nic *efx, 
	return rc;

}



/* Remove filters that weren't renewed.  Since nothing else changes the AUTO_OLD

 * flag or removes these filters, we don't need to hold the filter_lock while

 * scanning for these filters.

 */

/* Remove filters that weren't renewed. */

static void efx_ef10_filter_remove_old(struct efx_nic *efx)

{

	struct efx_ef10_filter_table *table = efx->filter_state;
@@ -5645,6 +5480,7 @@ static void efx_ef10_filter_remove_old(struct efx_nic *efx) 
	int rc;

	int i;



	down_write(&table->lock);

	for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) {

		if (READ_ONCE(table->entry[i].spec) &

		    EFX_EF10_FILTER_FLAG_AUTO_OLD) {
@@ -5656,6 +5492,7 @@ static void efx_ef10_filter_remove_old(struct efx_nic *efx) 
				remove_failed++;

		}

	}

	up_write(&table->lock);



	if (remove_failed)

		netif_info(efx, drv, efx->net_dev,
@@ -6784,7 +6621,6 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = { 
	.filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit,

	.filter_get_rx_ids = efx_ef10_filter_get_rx_ids,

#ifdef CONFIG_RFS_ACCEL

	.filter_rfs_insert = efx_ef10_filter_rfs_insert,

	.filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one,

#endif

#ifdef CONFIG_SFC_MTD
@@ -6897,7 +6733,6 @@ const struct efx_nic_type efx_hunt_a0_nic_type = { 
	.filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit,

	.filter_get_rx_ids = efx_ef10_filter_get_rx_ids,

#ifdef CONFIG_RFS_ACCEL

	.filter_rfs_insert = efx_ef10_filter_rfs_insert,

	.filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one,

#endif

#ifdef CONFIG_SFC_MTD

drivers/net/ethernet/sfc/efx.c

+26 −4

File changed.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/sfc/efx.h

+7 −4
Original line number Diff line number Diff line
@@ -170,22 +170,25 @@ static inline s32 efx_filter_get_rx_ids(struct efx_nic *efx, 
int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,

		   u16 rxq_index, u32 flow_id);

bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota);

static inline void efx_filter_rfs_expire(struct efx_channel *channel)

static inline void efx_filter_rfs_expire(struct work_struct *data)

{

	struct efx_channel *channel = container_of(data, struct efx_channel,

						   filter_work);



	if (channel->rfs_filters_added >= 60 &&

	    __efx_filter_rfs_expire(channel->efx, 100))

		channel->rfs_filters_added -= 60;

}

#define efx_filter_rfs_enabled() 1

#else

static inline void efx_filter_rfs_expire(struct efx_channel *channel) {}

static inline void efx_filter_rfs_expire(struct work_struct *data) {}

#define efx_filter_rfs_enabled() 0

#endif

bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec);



/* RSS contexts */

struct efx_rss_context *efx_alloc_rss_context_entry(struct list_head *list);

struct efx_rss_context *efx_find_rss_context_entry(u32 id, struct list_head *list);

struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx);

struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id);

void efx_free_rss_context_entry(struct efx_rss_context *ctx);

static inline bool efx_rss_active(struct efx_rss_context *ctx)

{

drivers/net/ethernet/sfc/ethtool.c

+49 −28

File changed.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/sfc/farch.c

+38 −42

File changed.

Preview size limit exceeded, changes collapsed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE