Commit 57d0a1c1 authored by Jesper Dangaard Brouer's avatar Jesper Dangaard Brouer Committed by David S. Miller
Browse files

xdp: allow page_pool as an allocator type in xdp_return_frame 



New allocator type MEM_TYPE_PAGE_POOL for page_pool usage.

The registered allocator page_pool pointer is not available directly
from xdp_rxq_info, but it could be (if needed).  For now, the driver
should keep separate track of the page_pool pointer, which it should
use for RX-ring page allocation.

As suggested by Saeed, to maintain a symmetric API it is the drivers
responsibility to allocate/create and free/destroy the page_pool.
Thus, after the driver have called xdp_rxq_info_unreg(), it is drivers
responsibility to free the page_pool, but with a RCU free call.  This
is done easily via the page_pool helper page_pool_destroy() (which
avoids touching any driver code during the RCU callback, which could
happen after the driver have been unloaded).

V8: address issues found by kbuild test robot
 - Address sparse should be static warnings
 - Allow xdp.o to be compiled without page_pool.o

V9: Remove inline from .c file, compiler knows best

Signed-off-by: default avatarJesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent ff7d6b27

include/net/page_pool.h

+14 −0
Original line number Diff line number Diff line
@@ -117,7 +117,12 @@ void __page_pool_put_page(struct page_pool *pool, 


static inline void page_pool_put_page(struct page_pool *pool, struct page *page)

{

	/* When page_pool isn't compiled-in, net/core/xdp.c doesn't

	 * allow registering MEM_TYPE_PAGE_POOL, but shield linker.

	 */

#ifdef CONFIG_PAGE_POOL

	__page_pool_put_page(pool, page, false);

#endif

}

/* Very limited use-cases allow recycle direct */

static inline void page_pool_recycle_direct(struct page_pool *pool,
@@ -126,4 +131,13 @@ static inline void page_pool_recycle_direct(struct page_pool *pool, 
	__page_pool_put_page(pool, page, true);

}



static inline bool is_page_pool_compiled_in(void)

{

#ifdef CONFIG_PAGE_POOL

	return true;

#else

	return false;

#endif

}



#endif /* _NET_PAGE_POOL_H */

include/net/xdp.h

+3 −0
Original line number Diff line number Diff line
@@ -36,6 +36,7 @@ 
enum xdp_mem_type {

	MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */

	MEM_TYPE_PAGE_ORDER0,     /* Orig XDP full page model */

	MEM_TYPE_PAGE_POOL,

	MEM_TYPE_MAX,

};
@@ -44,6 +45,8 @@ struct xdp_mem_info { 
	u32 id;

};



struct page_pool;



struct xdp_rxq_info {

	struct net_device *dev;

	u32 queue_index;

net/core/xdp.c

+48 −12
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
#include <linux/slab.h>

#include <linux/idr.h>

#include <linux/rhashtable.h>

#include <net/page_pool.h>



#include <net/xdp.h>
@@ -27,7 +28,10 @@ static struct rhashtable *mem_id_ht; 


struct xdp_mem_allocator {

	struct xdp_mem_info mem;

	union {

		void *allocator;

		struct page_pool *page_pool;

	};

	struct rhash_head node;

	struct rcu_head rcu;

};
@@ -74,7 +78,9 @@ static void __xdp_mem_allocator_rcu_free(struct rcu_head *rcu) 
	/* Allow this ID to be reused */

	ida_simple_remove(&mem_id_pool, xa->mem.id);



	/* TODO: Depending on allocator type/pointer free resources */

	/* Notice, driver is expected to free the *allocator,

	 * e.g. page_pool, and MUST also use RCU free.

	 */



	/* Poison memory */

	xa->mem.id = 0xFFFF;
@@ -225,6 +231,17 @@ again: 
	return id;

}



static bool __is_supported_mem_type(enum xdp_mem_type type)

{

	if (type == MEM_TYPE_PAGE_POOL)

		return is_page_pool_compiled_in();



	if (type >= MEM_TYPE_MAX)

		return false;



	return true;

}



int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,

			       enum xdp_mem_type type, void *allocator)

{
@@ -238,13 +255,16 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, 
		return -EFAULT;

	}



	if (type >= MEM_TYPE_MAX)

		return -EINVAL;

	if (!__is_supported_mem_type(type))

		return -EOPNOTSUPP;



	xdp_rxq->mem.type = type;



	if (!allocator)

	if (!allocator) {

		if (type == MEM_TYPE_PAGE_POOL)

			return -EINVAL; /* Setup time check page_pool req */

		return 0;

	}



	/* Delay init of rhashtable to save memory if feature isn't used */

	if (!mem_id_init) {
@@ -290,15 +310,31 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model); 


void xdp_return_frame(void *data, struct xdp_mem_info *mem)

{

	if (mem->type == MEM_TYPE_PAGE_SHARED) {

	struct xdp_mem_allocator *xa;

	struct page *page;



	switch (mem->type) {

	case MEM_TYPE_PAGE_POOL:

		rcu_read_lock();

		/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */

		xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);

		page = virt_to_head_page(data);

		if (xa)

			page_pool_put_page(xa->page_pool, page);

		else

			put_page(page);

		rcu_read_unlock();

		break;

	case MEM_TYPE_PAGE_SHARED:

		page_frag_free(data);

		return;

	}



	if (mem->type == MEM_TYPE_PAGE_ORDER0) {

		struct page *page = virt_to_page(data); /* Assumes order0 page*/



		break;

	case MEM_TYPE_PAGE_ORDER0:

		page = virt_to_page(data); /* Assumes order0 page*/

		put_page(page);

		break;

	default:

		/* Not possible, checked in xdp_rxq_info_reg_mem_model() */

		break;

	}

}

EXPORT_SYMBOL_GPL(xdp_return_frame);

CRA Git | Maintained and supported by SUSTech CRA and CCSE