qede: add .ndo_xdp_xmit() and XDP_REDIRECT support (d1b25b79) · Commits · 戴 / test

drivers/net/ethernet/qlogic/qede/qede.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -199,6 +199,7 @@ struct qede_dev {
		u8 fp_num_rx;
		u16 req_queues;
		u16 num_queues;
		u16 total_xdp_queues;

		#define QEDE_QUEUE_CNT(edev) ((edev)->num_queues)
		#define QEDE_RSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_tx)
		@@ -381,6 +382,7 @@ struct sw_tx_bd {

		struct sw_tx_xdp {
		struct page *page;
		struct xdp_frame *xdpf;
		dma_addr_t mapping;
		};

		@@ -403,6 +405,9 @@ struct qede_tx_queue {
		void __iomem *doorbell_addr;
		union db_prod tx_db;

		/* Spinlock for XDP queues in case of XDP_REDIRECT */
		spinlock_t xdp_tx_lock;

		int index; /* Slowpath only */
		#define QEDE_TXQ_XDP_TO_IDX(edev, txq) ((txq)->index - \
		QEDE_MAX_TSS_CNT(edev))
		@@ -456,6 +461,7 @@ struct qede_fastpath {

		u8 xdp_xmit;
		#define QEDE_XDP_TX BIT(0)
		#define QEDE_XDP_REDIRECT BIT(1)

		struct napi_struct napi;
		struct qed_sb_info *sb_info;
		@@ -516,6 +522,8 @@ struct qede_reload_args {

		/* Datapath functions definition */
		netdev_tx_t qede_start_xmit(struct sk_buff skb, struct net_device ndev);
		int qede_xdp_transmit(struct net_device *dev, int n_frames,
		struct xdp_frame **frames, u32 flags);
		u16 qede_select_queue(struct net_device dev, struct sk_buff skb,
		struct net_device *sb_dev);
		netdev_features_t qede_features_check(struct sk_buff *skb,

drivers/net/ethernet/qlogic/qede/qede_fp.c

+92 −5

Original line number	Diff line number	Diff line
		@@ -303,7 +303,7 @@ static inline void qede_update_tx_producer(struct qede_tx_queue *txq)
		}

		static int qede_xdp_xmit(struct qede_tx_queue *txq, dma_addr_t dma, u16 pad,
		u16 len, struct page *page)
		u16 len, struct page page, struct xdp_frame xdpf)
		{
		struct eth_tx_1st_bd *bd;
		struct sw_tx_xdp *xdp;
		@@ -330,12 +330,66 @@ static int qede_xdp_xmit(struct qede_tx_queue *txq, dma_addr_t dma, u16 pad,
		xdp = txq->sw_tx_ring.xdp + txq->sw_tx_prod;
		xdp->mapping = dma;
		xdp->page = page;
		xdp->xdpf = xdpf;

		txq->sw_tx_prod = (txq->sw_tx_prod + 1) % txq->num_tx_buffers;

		return 0;
		}

		int qede_xdp_transmit(struct net_device *dev, int n_frames,
		struct xdp_frame **frames, u32 flags)
		{
		struct qede_dev *edev = netdev_priv(dev);
		struct device *dmadev = &edev->pdev->dev;
		struct qede_tx_queue *xdp_tx;
		struct xdp_frame *xdpf;
		dma_addr_t mapping;
		int i, drops = 0;
		u16 xdp_prod;

		if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
		return -EINVAL;

		if (unlikely(!netif_running(dev)))
		return -ENETDOWN;

		i = smp_processor_id() % edev->total_xdp_queues;
		xdp_tx = edev->fp_array[i].xdp_tx;

		spin_lock(&xdp_tx->xdp_tx_lock);

		for (i = 0; i < n_frames; i++) {
		xdpf = frames[i];

		mapping = dma_map_single(dmadev, xdpf->data, xdpf->len,
		DMA_TO_DEVICE);
		if (unlikely(dma_mapping_error(dmadev, mapping))) {
		xdp_return_frame_rx_napi(xdpf);
		drops++;

		continue;
		}

		if (unlikely(qede_xdp_xmit(xdp_tx, mapping, 0, xdpf->len,
		NULL, xdpf))) {
		xdp_return_frame_rx_napi(xdpf);
		drops++;
		}
		}

		if (flags & XDP_XMIT_FLUSH) {
		xdp_prod = qed_chain_get_prod_idx(&xdp_tx->tx_pbl);

		xdp_tx->tx_db.data.bd_prod = cpu_to_le16(xdp_prod);
		qede_update_tx_producer(xdp_tx);
		}

		spin_unlock(&xdp_tx->xdp_tx_lock);

		return n_frames - drops;
		}

		int qede_txq_has_work(struct qede_tx_queue *txq)
		{
		u16 hw_bd_cons;
		@@ -353,6 +407,7 @@ static void qede_xdp_tx_int(struct qede_dev edev, struct qede_tx_queue txq)
		{
		struct sw_tx_xdp xdp_info, xdp_arr = txq->sw_tx_ring.xdp;
		struct device *dev = &edev->pdev->dev;
		struct xdp_frame *xdpf;
		u16 hw_bd_cons;

		hw_bd_cons = le16_to_cpu(*txq->hw_cons_ptr);
		@@ -360,10 +415,19 @@ static void qede_xdp_tx_int(struct qede_dev edev, struct qede_tx_queue txq)

		while (hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl)) {
		xdp_info = xdp_arr + txq->sw_tx_cons;
		xdpf = xdp_info->xdpf;

		if (xdpf) {
		dma_unmap_single(dev, xdp_info->mapping, xdpf->len,
		DMA_TO_DEVICE);
		xdp_return_frame(xdpf);

		xdp_info->xdpf = NULL;
		} else {
		dma_unmap_page(dev, xdp_info->mapping, PAGE_SIZE,
		DMA_BIDIRECTIONAL);
		__free_page(xdp_info->page);
		}

		qed_chain_consume(&txq->tx_pbl);
		txq->sw_tx_cons = (txq->sw_tx_cons + 1) % txq->num_tx_buffers;
		@@ -1065,7 +1129,8 @@ static bool qede_rx_xdp(struct qede_dev *edev,
		* throw current buffer, as replacement was already allocated.
		*/
		if (unlikely(qede_xdp_xmit(fp->xdp_tx, bd->mapping,
		data_offset, len, bd->data))) {
		data_offset, len, bd->data,
		NULL))) {
		dma_unmap_page(rxq->dev, bd->mapping, PAGE_SIZE,
		rxq->data_direction);
		__free_page(bd->data);
		@@ -1079,6 +1144,25 @@ static bool qede_rx_xdp(struct qede_dev *edev,
		}

		/* Regardless, we've consumed an Rx BD */
		qede_rx_bd_ring_consume(rxq);
		break;
		case XDP_REDIRECT:
		/* We need the replacement buffer before transmit. */
		if (unlikely(qede_alloc_rx_buffer(rxq, true))) {
		qede_recycle_rx_bd_ring(rxq, 1);

		trace_xdp_exception(edev->ndev, prog, act);
		break;
		}

		dma_unmap_page(rxq->dev, bd->mapping, PAGE_SIZE,
		rxq->data_direction);

		if (unlikely(xdp_do_redirect(edev->ndev, &xdp, prog)))
		DP_NOTICE(edev, "Failed to redirect the packet\n");
		else
		fp->xdp_xmit \|= QEDE_XDP_REDIRECT;

		qede_rx_bd_ring_consume(rxq);
		break;
		default:
		@@ -1387,6 +1471,9 @@ int qede_poll(struct napi_struct *napi, int budget)
		qede_update_tx_producer(fp->xdp_tx);
		}

		if (fp->xdp_xmit & QEDE_XDP_REDIRECT)
		xdp_do_flush_map();

		return rx_work_done;
		}

drivers/net/ethernet/qlogic/qede/qede_main.c

+18 −0

Original line number	Diff line number	Diff line
		@@ -672,6 +672,7 @@ static const struct net_device_ops qede_netdev_ops = {
		#ifdef CONFIG_RFS_ACCEL
		.ndo_rx_flow_steer = qede_rx_flow_steer,
		#endif
		.ndo_xdp_xmit = qede_xdp_transmit,
		.ndo_setup_tc = qede_setup_tc_offload,
		};

		@@ -712,6 +713,7 @@ static const struct net_device_ops qede_netdev_vf_xdp_ops = {
		.ndo_udp_tunnel_del = udp_tunnel_nic_del_port,
		.ndo_features_check = qede_features_check,
		.ndo_bpf = qede_xdp,
		.ndo_xdp_xmit = qede_xdp_transmit,
		};

		/* -------------------------------------------------------------------------
		@@ -1712,6 +1714,7 @@ static void qede_init_fp(struct qede_dev *edev)
		{
		int queue_id, rxq_index = 0, txq_index = 0;
		struct qede_fastpath *fp;
		bool init_xdp = false;

		for_each_queue(queue_id) {
		fp = &edev->fp_array[queue_id];
		@@ -1723,6 +1726,9 @@ static void qede_init_fp(struct qede_dev *edev)
		fp->xdp_tx->index = QEDE_TXQ_IDX_TO_XDP(edev,
		rxq_index);
		fp->xdp_tx->is_xdp = 1;

		spin_lock_init(&fp->xdp_tx->xdp_tx_lock);
		init_xdp = true;
		}

		if (fp->type & QEDE_FASTPATH_RX) {
		@@ -1738,6 +1744,13 @@ static void qede_init_fp(struct qede_dev *edev)
		/* Driver have no error path from here */
		WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev,
		fp->rxq->rxq_id) < 0);

		if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq,
		MEM_TYPE_PAGE_ORDER0,
		NULL)) {
		DP_NOTICE(edev,
		"Failed to register XDP memory model\n");
		}
		}

		if (fp->type & QEDE_FASTPATH_TX) {
		@@ -1763,6 +1776,11 @@ static void qede_init_fp(struct qede_dev *edev)
		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
		edev->ndev->name, queue_id);
		}

		if (init_xdp) {
		edev->total_xdp_queues = QEDE_RSS_COUNT(edev);
		DP_INFO(edev, "Total XDP queues: %u\n", edev->total_xdp_queues);
		}
		}

		static int qede_set_real_num_queues(struct qede_dev *edev)

Admin message