Commit 0536b852 authored by Björn Töpel's avatar Björn Töpel Committed by Alexei Starovoitov
Browse files

xdp: Simplify devmap cleanup 

After the RCU flavor consolidation [1], call_rcu() and
synchronize_rcu() waits for preempt-disable regions (NAPI) in addition
to the read-side critical sections. As a result of this, the cleanup
code in devmap can be simplified

* There is no longer a need to flush in __dev_map_entry_free, since we
  know that this has been done when the call_rcu() callback is
  triggered.

* When freeing the map, there is no need to explicitly wait for a
  flush. It's guaranteed to be done after the synchronize_rcu() call
  in dev_map_free(). The rcu_barrier() is still needed, so that the
  map is not freed prior the elements.

[1] https://lwn.net/Articles/777036/



Signed-off-by: default avatarBjörn Töpel <bjorn.topel@intel.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20191219061006.21980-2-bjorn.topel@gmail.com
parent 5bf2fc1f

kernel/bpf/devmap.c

+5 −38
Original line number Diff line number Diff line
@@ -201,7 +201,7 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 
static void dev_map_free(struct bpf_map *map)

{

	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);

	int i, cpu;

	int i;



	/* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,

	 * so the programs (can be more than one that used this map) were
@@ -221,18 +221,6 @@ static void dev_map_free(struct bpf_map *map) 
	/* Make sure prior __dev_map_entry_free() have completed. */

	rcu_barrier();



	/* To ensure all pending flush operations have completed wait for flush

	 * list to empty on _all_ cpus.

	 * Because the above synchronize_rcu() ensures the map is disconnected

	 * from the program we can assume no new items will be added.

	 */

	for_each_online_cpu(cpu) {

		struct list_head *flush_list = per_cpu_ptr(dtab->flush_list, cpu);



		while (!list_empty(flush_list))

			cond_resched();

	}



	if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) {

		for (i = 0; i < dtab->n_buckets; i++) {

			struct bpf_dtab_netdev *dev;
@@ -345,8 +333,7 @@ static int dev_map_hash_get_next_key(struct bpf_map *map, void *key, 
	return -ENOENT;

}



static int bq_xmit_all(struct xdp_bulk_queue *bq, u32 flags,

		       bool in_napi_ctx)

static int bq_xmit_all(struct xdp_bulk_queue *bq, u32 flags)

{

	struct bpf_dtab_netdev *obj = bq->obj;

	struct net_device *dev = obj->dev;
@@ -384,11 +371,7 @@ error: 
	for (i = 0; i < bq->count; i++) {

		struct xdp_frame *xdpf = bq->q[i];



		/* RX path under NAPI protection, can return frames faster */

		if (likely(in_napi_ctx))

		xdp_return_frame_rx_napi(xdpf);

		else

			xdp_return_frame(xdpf);

		drops++;

	}

	goto out;
@@ -409,7 +392,7 @@ void __dev_map_flush(struct bpf_map *map) 


	rcu_read_lock();

	list_for_each_entry_safe(bq, tmp, flush_list, flush_node)

		bq_xmit_all(bq, XDP_XMIT_FLUSH, true);

		bq_xmit_all(bq, XDP_XMIT_FLUSH);

	rcu_read_unlock();

}
@@ -440,7 +423,7 @@ static int bq_enqueue(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf, 
	struct xdp_bulk_queue *bq = this_cpu_ptr(obj->bulkq);



	if (unlikely(bq->count == DEV_MAP_BULK_SIZE))

		bq_xmit_all(bq, 0, true);

		bq_xmit_all(bq, 0);



	/* Ingress dev_rx will be the same for all xdp_frame's in

	 * bulk_queue, because bq stored per-CPU and must be flushed
@@ -509,27 +492,11 @@ static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key) 
	return dev ? &dev->ifindex : NULL;

}



static void dev_map_flush_old(struct bpf_dtab_netdev *dev)

{

	if (dev->dev->netdev_ops->ndo_xdp_xmit) {

		struct xdp_bulk_queue *bq;

		int cpu;



		rcu_read_lock();

		for_each_online_cpu(cpu) {

			bq = per_cpu_ptr(dev->bulkq, cpu);

			bq_xmit_all(bq, XDP_XMIT_FLUSH, false);

		}

		rcu_read_unlock();

	}

}



static void __dev_map_entry_free(struct rcu_head *rcu)

{

	struct bpf_dtab_netdev *dev;



	dev = container_of(rcu, struct bpf_dtab_netdev, rcu);

	dev_map_flush_old(dev);

	free_percpu(dev->bulkq);

	dev_put(dev->dev);

	kfree(dev);

CRA Git | Maintained and supported by SUSTech CRA and CCSE