Commit 74a1c059 authored by Ioana Ciornei's avatar Ioana Ciornei Committed by David S. Miller
Browse files

dpaa2-eth: add bulking to XDP_TX 



Add driver level bulking to the XDP_TX action.

An array of frame descriptors is held for each Tx frame queue and
populated accordingly when the action returned by the XDP program is
XDP_TX. The frames will be actually enqueued only when the array is
filled. At the end of the NAPI cycle a flush on the queued frames is
performed in order to enqueue the remaining FDs.

Signed-off-by: default avatarIoana Ciornei <ioana.ciornei@nxp.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 6f42a293

drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c

+45 −23
Original line number Diff line number Diff line
@@ -273,13 +273,43 @@ static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv, 
	return total_enqueued;

}



static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,

static void xdp_tx_flush(struct dpaa2_eth_priv *priv,

			 struct dpaa2_eth_channel *ch,

			 struct dpaa2_eth_fq *fq)

{

	struct rtnl_link_stats64 *percpu_stats;

	struct dpaa2_fd *fds;

	int enqueued, i;



	percpu_stats = this_cpu_ptr(priv->percpu_stats);



	// enqueue the array of XDP_TX frames

	enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds);



	/* update statistics */

	percpu_stats->tx_packets += enqueued;

	fds = fq->xdp_tx_fds.fds;

	for (i = 0; i < enqueued; i++) {

		percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);

		ch->stats.xdp_tx++;

	}

	for (i = enqueued; i < fq->xdp_tx_fds.num; i++) {

		xdp_release_buf(priv, ch, dpaa2_fd_get_addr(&fds[i]));

		percpu_stats->tx_errors++;

		ch->stats.xdp_tx_err++;

	}

	fq->xdp_tx_fds.num = 0;

}



static void xdp_enqueue(struct dpaa2_eth_priv *priv,

			struct dpaa2_eth_channel *ch,

			struct dpaa2_fd *fd,

			void *buf_start, u16 queue_id)

{

	struct dpaa2_eth_fq *fq;

	struct dpaa2_faead *faead;

	struct dpaa2_fd *dest_fd;

	struct dpaa2_eth_fq *fq;

	u32 ctrl, frc;

	int i, err;



	/* Mark the egress frame hardware annotation area as valid */

	frc = dpaa2_fd_get_frc(fd);
@@ -296,13 +326,13 @@ static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd, 
	faead->conf_fqid = 0;



	fq = &priv->fq[queue_id];

	for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {

		err = priv->enqueue(priv, fq, fd, 0, 1, NULL);

		if (err != -EBUSY)

			break;

	}

	dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++];

	memcpy(dest_fd, fd, sizeof(*dest_fd));



	return err;

	if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE)

		return;



	xdp_tx_flush(priv, ch, fq);

}



static u32 run_xdp(struct dpaa2_eth_priv *priv,
@@ -311,14 +341,11 @@ static u32 run_xdp(struct dpaa2_eth_priv *priv, 
		   struct dpaa2_fd *fd, void *vaddr)

{

	dma_addr_t addr = dpaa2_fd_get_addr(fd);

	struct rtnl_link_stats64 *percpu_stats;

	struct bpf_prog *xdp_prog;

	struct xdp_buff xdp;

	u32 xdp_act = XDP_PASS;

	int err;



	percpu_stats = this_cpu_ptr(priv->percpu_stats);



	rcu_read_lock();



	xdp_prog = READ_ONCE(ch->xdp.prog);
@@ -344,16 +371,7 @@ static u32 run_xdp(struct dpaa2_eth_priv *priv, 
	case XDP_PASS:

		break;

	case XDP_TX:

		err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid);

		if (err) {

			xdp_release_buf(priv, ch, addr);

			percpu_stats->tx_errors++;

			ch->stats.xdp_tx_err++;

		} else {

			percpu_stats->tx_packets++;

			percpu_stats->tx_bytes += dpaa2_fd_get_len(fd);

			ch->stats.xdp_tx++;

		}

		xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);

		break;

	default:

		bpf_warn_invalid_xdp_action(xdp_act);
@@ -1175,6 +1193,7 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 
	int store_cleaned, work_done;

	struct list_head rx_list;

	int retries = 0;

	u16 flowid;

	int err;



	ch = container_of(napi, struct dpaa2_eth_channel, napi);
@@ -1197,6 +1216,7 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 
			break;

		if (fq->type == DPAA2_RX_FQ) {

			rx_cleaned += store_cleaned;

			flowid = fq->flowid;

		} else {

			txconf_cleaned += store_cleaned;

			/* We have a single Tx conf FQ on this channel */
@@ -1239,6 +1259,8 @@ out: 


	if (ch->xdp.res & XDP_REDIRECT)

		xdp_do_flush_map();

	else if (rx_cleaned && ch->xdp.res & XDP_TX)

		xdp_tx_flush(priv, ch, &priv->fq[flowid]);



	return work_done;

}

drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h

+1 −0
Original line number Diff line number Diff line
@@ -334,6 +334,7 @@ struct dpaa2_eth_fq { 
	struct dpaa2_eth_fq_stats stats;



	struct dpaa2_eth_xdp_fds xdp_redirect_fds;

	struct dpaa2_eth_xdp_fds xdp_tx_fds;

};



struct dpaa2_eth_ch_xdp {

CRA Git | Maintained and supported by SUSTech CRA and CCSE