Commit f8d62037 authored by Edward Cree's avatar Edward Cree Committed by David S. Miller
Browse files

sfc: ARFS filter IDs 



Associate an arbitrary ID with each ARFS filter, allowing to properly query
 for expiry.  The association is maintained in a hash table, which is
 protected by a spinlock.

v3: fix build warnings when CONFIG_RFS_ACCEL is disabled (thanks lkp-robot).
v2: fixed uninitialised variable (thanks davem and lkp-robot).

Fixes: 3af0f342 ("sfc: replace asynchronous filter operations")
Signed-off-by: default avatarEdward Cree <ecree@solarflare.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent d805c520

drivers/net/ethernet/sfc/ef10.c

+46 −34
Original line number Diff line number Diff line
@@ -3999,29 +3999,6 @@ static void efx_ef10_prepare_flr(struct efx_nic *efx) 
	atomic_set(&efx->active_queues, 0);

}



static bool efx_ef10_filter_equal(const struct efx_filter_spec *left,

				  const struct efx_filter_spec *right)

{

	if ((left->match_flags ^ right->match_flags) |

	    ((left->flags ^ right->flags) &

	     (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))

		return false;



	return memcmp(&left->outer_vid, &right->outer_vid,

		      sizeof(struct efx_filter_spec) -

		      offsetof(struct efx_filter_spec, outer_vid)) == 0;

}



static unsigned int efx_ef10_filter_hash(const struct efx_filter_spec *spec)

{

	BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3);

	return jhash2((const u32 *)&spec->outer_vid,

		      (sizeof(struct efx_filter_spec) -

		       offsetof(struct efx_filter_spec, outer_vid)) / 4,

		      0);

	/* XXX should we randomise the initval? */

}



/* Decide whether a filter should be exclusive or else should allow

 * delivery to additional recipients.  Currently we decide that

 * filters for specific local unicast MAC and IP addresses are
@@ -4346,7 +4323,7 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, 
		goto out_unlock;

	match_pri = rc;



	hash = efx_ef10_filter_hash(spec);

	hash = efx_filter_spec_hash(spec);

	is_mc_recip = efx_filter_is_mc_recipient(spec);

	if (is_mc_recip)

		bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
@@ -4378,7 +4355,7 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, 
		if (!saved_spec) {

			if (ins_index < 0)

				ins_index = i;

		} else if (efx_ef10_filter_equal(spec, saved_spec)) {

		} else if (efx_filter_spec_equal(spec, saved_spec)) {

			if (spec->priority < saved_spec->priority &&

			    spec->priority != EFX_FILTER_PRI_AUTO) {

				rc = -EPERM;
@@ -4762,27 +4739,62 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx, 
static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,

					   unsigned int filter_idx)

{

	struct efx_filter_spec *spec, saved_spec;

	struct efx_ef10_filter_table *table;

	struct efx_filter_spec *spec;

	bool ret;

	struct efx_arfs_rule *rule = NULL;

	bool ret = true, force = false;

	u16 arfs_id;



	down_read(&efx->filter_sem);

	table = efx->filter_state;

	down_write(&table->lock);

	spec = efx_ef10_filter_entry_spec(table, filter_idx);



	if (!spec || spec->priority != EFX_FILTER_PRI_HINT) {

		ret = true;

	if (!spec || spec->priority != EFX_FILTER_PRI_HINT)

		goto out_unlock;

	}



	if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, 0)) {

		ret = false;

	spin_lock_bh(&efx->rps_hash_lock);

	if (!efx->rps_hash_table) {

		/* In the absence of the table, we always return 0 to ARFS. */

		arfs_id = 0;

	} else {

		rule = efx_rps_hash_find(efx, spec);

		if (!rule)

			/* ARFS table doesn't know of this filter, so remove it */

			goto expire;

		arfs_id = rule->arfs_id;

		ret = efx_rps_check_rule(rule, filter_idx, &force);

		if (force)

			goto expire;

		if (!ret) {

			spin_unlock_bh(&efx->rps_hash_lock);

			goto out_unlock;

		}



	}

	if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id))

		ret = false;

	else if (rule)

		rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;

expire:

	saved_spec = *spec; /* remove operation will kfree spec */

	spin_unlock_bh(&efx->rps_hash_lock);

	/* At this point (since we dropped the lock), another thread might queue

	 * up a fresh insertion request (but the actual insertion will be held

	 * up by our possession of the filter table lock).  In that case, it

	 * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that

	 * the rule is not removed by efx_rps_hash_del() below.

	 */

	ret = efx_ef10_filter_remove_internal(efx, 1U << spec->priority,

					      filter_idx, true) == 0;

	/* While we can't safely dereference rule (we dropped the lock), we can

	 * still test it for NULL.

	 */

	if (ret && rule) {

		/* Expiring, so remove entry from ARFS table */

		spin_lock_bh(&efx->rps_hash_lock);

		efx_rps_hash_del(efx, &saved_spec);

		spin_unlock_bh(&efx->rps_hash_lock);

	}

out_unlock:

	up_write(&table->lock);

	up_read(&efx->filter_sem);

drivers/net/ethernet/sfc/efx.c

+143 −0
Original line number Diff line number Diff line
@@ -3027,6 +3027,10 @@ static int efx_init_struct(struct efx_nic *efx, 
	mutex_init(&efx->mac_lock);

#ifdef CONFIG_RFS_ACCEL

	mutex_init(&efx->rps_mutex);

	spin_lock_init(&efx->rps_hash_lock);

	/* Failure to allocate is not fatal, but may degrade ARFS performance */

	efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE,

				      sizeof(*efx->rps_hash_table), GFP_KERNEL);

#endif

	efx->phy_op = &efx_dummy_phy_operations;

	efx->mdio.dev = net_dev;
@@ -3070,6 +3074,10 @@ static void efx_fini_struct(struct efx_nic *efx) 
{

	int i;



#ifdef CONFIG_RFS_ACCEL

	kfree(efx->rps_hash_table);

#endif



	for (i = 0; i < EFX_MAX_CHANNELS; i++)

		kfree(efx->channel[i]);
@@ -3092,6 +3100,141 @@ void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 
	stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);

}



bool efx_filter_spec_equal(const struct efx_filter_spec *left,

			   const struct efx_filter_spec *right)

{

	if ((left->match_flags ^ right->match_flags) |

	    ((left->flags ^ right->flags) &

	     (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))

		return false;



	return memcmp(&left->outer_vid, &right->outer_vid,

		      sizeof(struct efx_filter_spec) -

		      offsetof(struct efx_filter_spec, outer_vid)) == 0;

}



u32 efx_filter_spec_hash(const struct efx_filter_spec *spec)

{

	BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3);

	return jhash2((const u32 *)&spec->outer_vid,

		      (sizeof(struct efx_filter_spec) -

		       offsetof(struct efx_filter_spec, outer_vid)) / 4,

		      0);

}



#ifdef CONFIG_RFS_ACCEL

bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,

			bool *force)

{

	if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {

		/* ARFS is currently updating this entry, leave it */

		return false;

	}

	if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {

		/* ARFS tried and failed to update this, so it's probably out

		 * of date.  Remove the filter and the ARFS rule entry.

		 */

		rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;

		*force = true;

		return true;

	} else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */

		/* ARFS has moved on, so old filter is not needed.  Since we did

		 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will

		 * not be removed by efx_rps_hash_del() subsequently.

		 */

		*force = true;

		return true;

	}

	/* Remove it iff ARFS wants to. */

	return true;

}



struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,

				       const struct efx_filter_spec *spec)

{

	u32 hash = efx_filter_spec_hash(spec);



	WARN_ON(!spin_is_locked(&efx->rps_hash_lock));

	if (!efx->rps_hash_table)

		return NULL;

	return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];

}



struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,

					const struct efx_filter_spec *spec)

{

	struct efx_arfs_rule *rule;

	struct hlist_head *head;

	struct hlist_node *node;



	head = efx_rps_hash_bucket(efx, spec);

	if (!head)

		return NULL;

	hlist_for_each(node, head) {

		rule = container_of(node, struct efx_arfs_rule, node);

		if (efx_filter_spec_equal(spec, &rule->spec))

			return rule;

	}

	return NULL;

}



struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,

				       const struct efx_filter_spec *spec,

				       bool *new)

{

	struct efx_arfs_rule *rule;

	struct hlist_head *head;

	struct hlist_node *node;



	head = efx_rps_hash_bucket(efx, spec);

	if (!head)

		return NULL;

	hlist_for_each(node, head) {

		rule = container_of(node, struct efx_arfs_rule, node);

		if (efx_filter_spec_equal(spec, &rule->spec)) {

			*new = false;

			return rule;

		}

	}

	rule = kmalloc(sizeof(*rule), GFP_ATOMIC);

	*new = true;

	if (rule) {

		memcpy(&rule->spec, spec, sizeof(rule->spec));

		hlist_add_head(&rule->node, head);

	}

	return rule;

}



void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec)

{

	struct efx_arfs_rule *rule;

	struct hlist_head *head;

	struct hlist_node *node;



	head = efx_rps_hash_bucket(efx, spec);

	if (WARN_ON(!head))

		return;

	hlist_for_each(node, head) {

		rule = container_of(node, struct efx_arfs_rule, node);

		if (efx_filter_spec_equal(spec, &rule->spec)) {

			/* Someone already reused the entry.  We know that if

			 * this check doesn't fire (i.e. filter_id == REMOVING)

			 * then the REMOVING mark was put there by our caller,

			 * because caller is holding a lock on filter table and

			 * only holders of that lock set REMOVING.

			 */

			if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)

				return;

			hlist_del(node);

			kfree(rule);

			return;

		}

	}

	/* We didn't find it. */

	WARN_ON(1);

}

#endif



/* RSS contexts.  We're using linked lists and crappy O(n) algorithms, because

 * (a) this is an infrequent control-plane operation and (b) n is small (max 64)

 */

drivers/net/ethernet/sfc/efx.h

+21 −0
Original line number Diff line number Diff line
@@ -186,6 +186,27 @@ static inline void efx_filter_rfs_expire(struct work_struct *data) {} 
#endif

bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec);



bool efx_filter_spec_equal(const struct efx_filter_spec *left,

			   const struct efx_filter_spec *right);

u32 efx_filter_spec_hash(const struct efx_filter_spec *spec);



#ifdef CONFIG_RFS_ACCEL

bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,

			bool *force);



struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,

					const struct efx_filter_spec *spec);



/* @new is written to indicate if entry was newly added (true) or if an old

 * entry was found and returned (false).

 */

struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,

				       const struct efx_filter_spec *spec,

				       bool *new);



void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec);

#endif



/* RSS contexts */

struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx);

struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id);

drivers/net/ethernet/sfc/farch.c

+34 −7
Original line number Diff line number Diff line
@@ -2905,18 +2905,45 @@ bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, 
{

	struct efx_farch_filter_state *state = efx->filter_state;

	struct efx_farch_filter_table *table;

	bool ret = false;

	bool ret = false, force = false;

	u16 arfs_id;



	down_write(&state->lock);

	spin_lock_bh(&efx->rps_hash_lock);

	table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP];

	if (test_bit(index, table->used_bitmap) &&

	    table->spec[index].priority == EFX_FILTER_PRI_HINT &&

	    rps_may_expire_flow(efx->net_dev, table->spec[index].dmaq_id,

				flow_id, 0)) {

	    table->spec[index].priority == EFX_FILTER_PRI_HINT) {

		struct efx_arfs_rule *rule = NULL;

		struct efx_filter_spec spec;



		efx_farch_filter_to_gen_spec(&spec, &table->spec[index]);

		if (!efx->rps_hash_table) {

			/* In the absence of the table, we always returned 0 to

			 * ARFS, so use the same to query it.

			 */

			arfs_id = 0;

		} else {

			rule = efx_rps_hash_find(efx, &spec);

			if (!rule) {

				/* ARFS table doesn't know of this filter, remove it */

				force = true;

			} else {

				arfs_id = rule->arfs_id;

				if (!efx_rps_check_rule(rule, index, &force))

					goto out_unlock;

			}

		}

		if (force || rps_may_expire_flow(efx->net_dev, spec.dmaq_id,

						 flow_id, arfs_id)) {

			if (rule)

				rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;

			efx_rps_hash_del(efx, &spec);

			efx_farch_filter_table_clear_entry(efx, table, index);

			ret = true;

		}



	}

out_unlock:

	spin_unlock_bh(&efx->rps_hash_lock);

	up_write(&state->lock);

	return ret;

}

drivers/net/ethernet/sfc/net_driver.h

+36 −0
Original line number Diff line number Diff line
@@ -734,6 +734,35 @@ struct efx_rss_context { 
};



#ifdef CONFIG_RFS_ACCEL

/* Order of these is important, since filter_id >= %EFX_ARFS_FILTER_ID_PENDING

 * is used to test if filter does or will exist.

 */

#define EFX_ARFS_FILTER_ID_PENDING	-1

#define EFX_ARFS_FILTER_ID_ERROR	-2

#define EFX_ARFS_FILTER_ID_REMOVING	-3

/**

 * struct efx_arfs_rule - record of an ARFS filter and its IDs

 * @node: linkage into hash table

 * @spec: details of the filter (used as key for hash table).  Use efx->type to

 *	determine which member to use.

 * @rxq_index: channel to which the filter will steer traffic.

 * @arfs_id: filter ID which was returned to ARFS

 * @filter_id: index in software filter table.  May be

 *	%EFX_ARFS_FILTER_ID_PENDING if filter was not inserted yet,

 *	%EFX_ARFS_FILTER_ID_ERROR if filter insertion failed, or

 *	%EFX_ARFS_FILTER_ID_REMOVING if expiry is currently removing the filter.

 */

struct efx_arfs_rule {

	struct hlist_node node;

	struct efx_filter_spec spec;

	u16 rxq_index;

	u16 arfs_id;

	s32 filter_id;

};



/* Size chosen so that the table is one page (4kB) */

#define EFX_ARFS_HASH_TABLE_SIZE	512



/**

 * struct efx_async_filter_insertion - Request to asynchronously insert a filter

 * @net_dev: Reference to the netdevice
@@ -873,6 +902,10 @@ struct efx_async_filter_insertion { 
 *	@rps_expire_channel's @rps_flow_id

 * @rps_slot_map: bitmap of in-flight entries in @rps_slot

 * @rps_slot: array of ARFS insertion requests for efx_filter_rfs_work()

 * @rps_hash_lock: Protects ARFS filter mapping state (@rps_hash_table and

 *	@rps_next_id).

 * @rps_hash_table: Mapping between ARFS filters and their various IDs

 * @rps_next_id: next arfs_id for an ARFS filter

 * @active_queues: Count of RX and TX queues that haven't been flushed and drained.

 * @rxq_flush_pending: Count of number of receive queues that need to be flushed.

 *	Decremented when the efx_flush_rx_queue() is called.
@@ -1029,6 +1062,9 @@ struct efx_nic { 
	unsigned int rps_expire_index;

	unsigned long rps_slot_map;

	struct efx_async_filter_insertion rps_slot[EFX_RPS_MAX_IN_FLIGHT];

	spinlock_t rps_hash_lock;

	struct hlist_head *rps_hash_table;

	u32 rps_next_id;

#endif



	atomic_t active_queues;

CRA Git | Maintained and supported by SUSTech CRA and CCSE