Commit ca0e0146 authored by Lorenzo Bianconi's avatar Lorenzo Bianconi Committed by David S. Miller
Browse files

net: mvneta: move skb build after descriptors processing 



Move skb build after all descriptors processing. This is a preliminary
patch to enable multi-buffers and JUMBO frames support for XDP.
Introduce mvneta_xdp_put_buff routine to release all pages used by a
XDP multi-buffer

Signed-off-by: default avatarLorenzo Bianconi <lorenzo@kernel.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 2f0bc54b

drivers/net/ethernet/marvell/mvneta.c

+101 −57
Original line number Diff line number Diff line
@@ -2026,6 +2026,20 @@ int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) 
	return i;

}



static void

mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,

		    struct xdp_buff *xdp, int sync_len, bool napi)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);

	int i;



	page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data),

			   sync_len, napi);

	for (i = 0; i < sinfo->nr_frags; i++)

		page_pool_put_full_page(rxq->page_pool,

					skb_frag_page(&sinfo->frags[i]), napi);

}



static int

mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,

			struct xdp_frame *xdpf, bool dma_map)
@@ -2229,6 +2243,7 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, 
	int data_len = -MVNETA_MH_SIZE, len;

	struct net_device *dev = pp->dev;

	enum dma_data_direction dma_dir;

	struct skb_shared_info *sinfo;

	int ret = 0;



	if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) {
@@ -2252,35 +2267,13 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, 
	xdp->data_end = xdp->data + data_len;

	xdp_set_data_meta_invalid(xdp);



	if (xdp_prog) {

		ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp, stats);

		if (ret)

			goto out;

	}



	rxq->skb = build_skb(xdp->data_hard_start, PAGE_SIZE);

	if (unlikely(!rxq->skb)) {

		struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);



		netdev_err(dev, "Can't allocate skb on queue %d\n", rxq->id);



		u64_stats_update_begin(&stats->syncp);

		stats->es.skb_alloc_error++;

		stats->rx_dropped++;

		u64_stats_update_end(&stats->syncp);



		return -ENOMEM;

	}

	page_pool_release_page(rxq->page_pool, page);

	sinfo = xdp_get_shared_info_from_buff(xdp);

	sinfo->nr_frags = 0;



	skb_reserve(rxq->skb,

		    xdp->data - xdp->data_hard_start);

	skb_put(rxq->skb, xdp->data_end - xdp->data);

	mvneta_rx_csum(pp, rx_desc->status, rxq->skb);

	if (xdp_prog)

		ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp, stats);



	rxq->left_size = rx_desc->data_size - len;



out:

	rx_desc->buf_phys_addr = 0;



	return ret;
@@ -2290,8 +2283,10 @@ static void 
mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,

			    struct mvneta_rx_desc *rx_desc,

			    struct mvneta_rx_queue *rxq,

			    struct xdp_buff *xdp,

			    struct page *page)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);

	struct net_device *dev = pp->dev;

	enum dma_data_direction dma_dir;

	int data_len, len;
@@ -2307,18 +2302,53 @@ mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, 
	dma_sync_single_for_cpu(dev->dev.parent,

				rx_desc->buf_phys_addr,

				len, dma_dir);

	if (data_len > 0) {

		/* refill descriptor with new buffer later */

		skb_add_rx_frag(rxq->skb,

				skb_shinfo(rxq->skb)->nr_frags,

				page, pp->rx_offset_correction, data_len,

				PAGE_SIZE);

	}

	page_pool_release_page(rxq->page_pool, page);



	if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) {

		skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags];



		skb_frag_off_set(frag, pp->rx_offset_correction);

		skb_frag_size_set(frag, data_len);

		__skb_frag_set_page(frag, page);

		sinfo->nr_frags++;



		rx_desc->buf_phys_addr = 0;

	}

	rxq->left_size -= len;

}



static struct sk_buff *

mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,

		      struct xdp_buff *xdp, u32 desc_status)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);

	int i, num_frags = sinfo->nr_frags;

	skb_frag_t frags[MAX_SKB_FRAGS];

	struct sk_buff *skb;



	memcpy(frags, sinfo->frags, sizeof(skb_frag_t) * num_frags);



	skb = build_skb(xdp->data_hard_start, PAGE_SIZE);

	if (!skb)

		return ERR_PTR(-ENOMEM);



	page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data));



	skb_reserve(skb, xdp->data - xdp->data_hard_start);

	skb_put(skb, xdp->data_end - xdp->data);

	mvneta_rx_csum(pp, desc_status, skb);



	for (i = 0; i < num_frags; i++) {

		struct page *page = skb_frag_page(&frags[i]);



		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,

				page, skb_frag_off(&frags[i]),

				skb_frag_size(&frags[i]), PAGE_SIZE);

		page_pool_release_page(rxq->page_pool, page);

	}



	return skb;

}



/* Main rx processing when using software buffer management */

static int mvneta_rx_swbm(struct napi_struct *napi,

			  struct mvneta_port *pp, int budget,
@@ -2326,22 +2356,25 @@ static int mvneta_rx_swbm(struct napi_struct *napi, 
{

	int rx_proc = 0, rx_todo, refill;

	struct net_device *dev = pp->dev;

	struct xdp_buff xdp_buf = {

		.frame_sz = PAGE_SIZE,

		.rxq = &rxq->xdp_rxq,

	};

	struct mvneta_stats ps = {};

	struct bpf_prog *xdp_prog;

	struct xdp_buff xdp_buf;

	u32 desc_status;



	/* Get number of received packets */

	rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);



	rcu_read_lock();

	xdp_prog = READ_ONCE(pp->xdp_prog);

	xdp_buf.rxq = &rxq->xdp_rxq;

	xdp_buf.frame_sz = PAGE_SIZE;



	/* Fairness NAPI loop */

	while (rx_proc < budget && rx_proc < rx_todo) {

		struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);

		u32 rx_status, index;

		struct sk_buff *skb;

		struct page *page;



		index = rx_desc - rxq->descs;
@@ -2357,21 +2390,21 @@ static int mvneta_rx_swbm(struct napi_struct *napi, 
			/* Check errors only for FIRST descriptor */

			if (rx_status & MVNETA_RXD_ERR_SUMMARY) {

				mvneta_rx_error(pp, rx_desc);

				/* leave the descriptor untouched */

				continue;

				goto next;

			}



			err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,

						   xdp_prog, page, &ps);

			if (err)

				continue;



			desc_status = rx_desc->status;

		} else {

			if (unlikely(!rxq->skb)) {

				pr_debug("no skb for rx_status 0x%x\n",

					 rx_status);

			if (unlikely(!xdp_buf.data_hard_start))

				continue;

			}

			mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, page);



			mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf,

						    page);

		} /* Middle or Last descriptor */



		if (!(rx_status & MVNETA_RXD_LAST_DESC))
@@ -2379,27 +2412,38 @@ static int mvneta_rx_swbm(struct napi_struct *napi, 
			continue;



		if (rxq->left_size) {

			pr_err("get last desc, but left_size (%d) != 0\n",

			       rxq->left_size);

			dev_kfree_skb_any(rxq->skb);

			rxq->left_size = 0;

			rxq->skb = NULL;

			continue;

			mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);

			goto next;

		}



		ps.rx_bytes += rxq->skb->len;

		ps.rx_packets++;

		skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status);

		if (IS_ERR(skb)) {

			struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);



		/* Linux processing */

		rxq->skb->protocol = eth_type_trans(rxq->skb, dev);

			mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);



		napi_gro_receive(napi, rxq->skb);

			u64_stats_update_begin(&stats->syncp);

			stats->es.skb_alloc_error++;

			stats->rx_dropped++;

			u64_stats_update_end(&stats->syncp);



		/* clean uncomplete skb pointer in queue */

		rxq->skb = NULL;

			goto next;

		}



		ps.rx_bytes += skb->len;

		ps.rx_packets++;



		skb->protocol = eth_type_trans(skb, dev);

		napi_gro_receive(napi, skb);

next:

		xdp_buf.data_hard_start = NULL;

	}

	rcu_read_unlock();



	if (xdp_buf.data_hard_start)

		mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);



	if (ps.xdp_redirect)

		xdp_do_flush_map();

CRA Git | Maintained and supported by SUSTech CRA and CCSE