Merge branch 'thunderx-xdp' (b0e92279) · Commits · 戴 / test

drivers/net/ethernet/cavium/thunder/nic.h

+8 −2

Original line number	Diff line number	Diff line
		@@ -252,12 +252,14 @@ struct nicvf_drv_stats {
		u64 tx_csum_overflow;

		/* driver debug stats */
		u64 rcv_buffer_alloc_failures;
		u64 tx_tso;
		u64 tx_timeout;
		u64 txq_stop;
		u64 txq_wake;

		u64 rcv_buffer_alloc_failures;
		u64 page_alloc;

		struct u64_stats_sync syncp;
		};

		@@ -266,9 +268,9 @@ struct nicvf {
		struct net_device *netdev;
		struct pci_dev *pdev;
		void __iomem *reg_base;
		struct bpf_prog *xdp_prog;
		#define MAX_QUEUES_PER_QSET 8
		struct queue_set *qs;
		struct nicvf_cq_poll *napi[8];
		void *iommu_domain;
		u8 vf_id;
		u8 sqs_id;
		@@ -294,6 +296,7 @@ struct nicvf {
		/* Queue count */
		u8 rx_queues;
		u8 tx_queues;
		u8 xdp_tx_queues;
		u8 max_queues;

		u8 node;
		@@ -318,6 +321,9 @@ struct nicvf {
		struct nicvf_drv_stats __percpu *drv_stats;
		struct bgx_stats bgx_stats;

		/* Napi */
		struct nicvf_cq_poll *napi[8];

		/* MSI-X */
		u8 num_vec;
		char irq_name[NIC_VF_MSIX_VECTORS][IFNAMSIZ + 15];

drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c

+22 −7

Original line number	Diff line number	Diff line
		@@ -100,11 +100,12 @@ static const struct nicvf_stat nicvf_drv_stats[] = {
		NICVF_DRV_STAT(tx_csum_overlap),
		NICVF_DRV_STAT(tx_csum_overflow),

		NICVF_DRV_STAT(rcv_buffer_alloc_failures),
		NICVF_DRV_STAT(tx_tso),
		NICVF_DRV_STAT(tx_timeout),
		NICVF_DRV_STAT(txq_stop),
		NICVF_DRV_STAT(txq_wake),
		NICVF_DRV_STAT(rcv_buffer_alloc_failures),
		NICVF_DRV_STAT(page_alloc),
		};

		static const struct nicvf_stat nicvf_queue_stats[] = {
		@@ -720,7 +721,7 @@ static int nicvf_set_channels(struct net_device *dev,
		struct nicvf *nic = netdev_priv(dev);
		int err = 0;
		bool if_up = netif_running(dev);
		int cqcount;
		u8 cqcount, txq_count;

		if (!channel->rx_count \|\| !channel->tx_count)
		return -EINVAL;
		@@ -729,10 +730,26 @@ static int nicvf_set_channels(struct net_device *dev,
		if (channel->tx_count > nic->max_queues)
		return -EINVAL;

		if (nic->xdp_prog &&
		((channel->tx_count + channel->rx_count) > nic->max_queues)) {
		netdev_err(nic->netdev,
		"XDP mode, RXQs + TXQs > Max %d\n",
		nic->max_queues);
		return -EINVAL;
		}

		if (if_up)
		nicvf_stop(dev);

		cqcount = max(channel->rx_count, channel->tx_count);
		nic->rx_queues = channel->rx_count;
		nic->tx_queues = channel->tx_count;
		if (!nic->xdp_prog)
		nic->xdp_tx_queues = 0;
		else
		nic->xdp_tx_queues = channel->rx_count;

		txq_count = nic->xdp_tx_queues + nic->tx_queues;
		cqcount = max(nic->rx_queues, txq_count);

		if (cqcount > MAX_CMP_QUEUES_PER_QS) {
		nic->sqs_count = roundup(cqcount, MAX_CMP_QUEUES_PER_QS);
		@@ -741,12 +758,10 @@ static int nicvf_set_channels(struct net_device *dev,
		nic->sqs_count = 0;
		}

		nic->qs->rq_cnt = min_t(u32, channel->rx_count, MAX_RCV_QUEUES_PER_QS);
		nic->qs->sq_cnt = min_t(u32, channel->tx_count, MAX_SND_QUEUES_PER_QS);
		nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS);
		nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS);
		nic->qs->cq_cnt = max(nic->qs->rq_cnt, nic->qs->sq_cnt);

		nic->rx_queues = channel->rx_count;
		nic->tx_queues = channel->tx_count;
		err = nicvf_set_real_num_queues(dev, nic->tx_queues, nic->rx_queues);
		if (err)
		return err;

drivers/net/ethernet/cavium/thunder/nicvf_main.c

+276 −37

Original line number	Diff line number	Diff line
		@@ -17,6 +17,9 @@
		#include <linux/prefetch.h>
		#include <linux/irq.h>
		#include <linux/iommu.h>
		#include <linux/bpf.h>
		#include <linux/bpf_trace.h>
		#include <linux/filter.h>

		#include "nic_reg.h"
		#include "nic.h"
		@@ -397,8 +400,10 @@ static void nicvf_request_sqs(struct nicvf *nic)

		if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
		rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
		if (nic->tx_queues > MAX_SND_QUEUES_PER_QS)
		tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS;

		tx_queues = nic->tx_queues + nic->xdp_tx_queues;
		if (tx_queues > MAX_SND_QUEUES_PER_QS)
		tx_queues = tx_queues - MAX_SND_QUEUES_PER_QS;

		/* Set no of Rx/Tx queues in each of the SQsets */
		for (sqs = 0; sqs < nic->sqs_count; sqs++) {
		@@ -496,12 +501,99 @@ static int nicvf_init_resources(struct nicvf *nic)
		return 0;
		}

		static inline bool nicvf_xdp_rx(struct nicvf nic, struct bpf_prog prog,
		struct cqe_rx_t cqe_rx, struct snd_queue sq,
		struct sk_buff **skb)
		{
		struct xdp_buff xdp;
		struct page *page;
		u32 action;
		u16 len, offset = 0;
		u64 dma_addr, cpu_addr;
		void *orig_data;

		/* Retrieve packet buffer's DMA address and length */
		len = ((u16 )((void )cqe_rx + (3 sizeof(u64))));
		dma_addr = ((u64 )((void )cqe_rx + (7 sizeof(u64))));

		cpu_addr = nicvf_iova_to_phys(nic, dma_addr);
		if (!cpu_addr)
		return false;
		cpu_addr = (u64)phys_to_virt(cpu_addr);
		page = virt_to_page((void *)cpu_addr);

		xdp.data_hard_start = page_address(page);
		xdp.data = (void *)cpu_addr;
		xdp.data_end = xdp.data + len;
		orig_data = xdp.data;

		rcu_read_lock();
		action = bpf_prog_run_xdp(prog, &xdp);
		rcu_read_unlock();

		/* Check if XDP program has changed headers */
		if (orig_data != xdp.data) {
		len = xdp.data_end - xdp.data;
		offset = orig_data - xdp.data;
		dma_addr -= offset;
		}

		switch (action) {
		case XDP_PASS:
		/* Check if it's a recycled page, if not
		* unmap the DMA mapping.
		*
		* Recycled page holds an extra reference.
		*/
		if (page_ref_count(page) == 1) {
		dma_addr &= PAGE_MASK;
		dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
		RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
		DMA_FROM_DEVICE,
		DMA_ATTR_SKIP_CPU_SYNC);
		}

		/* Build SKB and pass on packet to network stack */
		*skb = build_skb(xdp.data,
		RCV_FRAG_LEN - cqe_rx->align_pad + offset);
		if (!*skb)
		put_page(page);
		else
		skb_put(*skb, len);
		return false;
		case XDP_TX:
		nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len);
		return true;
		default:
		bpf_warn_invalid_xdp_action(action);
		case XDP_ABORTED:
		trace_xdp_exception(nic->netdev, prog, action);
		case XDP_DROP:
		/* Check if it's a recycled page, if not
		* unmap the DMA mapping.
		*
		* Recycled page holds an extra reference.
		*/
		if (page_ref_count(page) == 1) {
		dma_addr &= PAGE_MASK;
		dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
		RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
		DMA_FROM_DEVICE,
		DMA_ATTR_SKIP_CPU_SYNC);
		}
		put_page(page);
		return true;
		}
		return false;
		}

		static void nicvf_snd_pkt_handler(struct net_device *netdev,
		struct cqe_send_t *cqe_tx,
		int cqe_type, int budget,
		int budget, int *subdesc_cnt,
		unsigned int tx_pkts, unsigned int tx_bytes)
		{
		struct sk_buff *skb = NULL;
		struct page *page;
		struct nicvf *nic = netdev_priv(netdev);
		struct snd_queue *sq;
		struct sq_hdr_subdesc *hdr;
		@@ -513,12 +605,26 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
		if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
		return;

		netdev_dbg(nic->netdev,
		"%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n",
		__func__, cqe_tx->sq_qs, cqe_tx->sq_idx,
		cqe_tx->sqe_ptr, hdr->subdesc_cnt);
		/* Check for errors */
		if (cqe_tx->send_status)
		nicvf_check_cqe_tx_errs(nic->pnicvf, cqe_tx);

		/* Is this a XDP designated Tx queue */
		if (sq->is_xdp) {
		page = (struct page *)sq->xdp_page[cqe_tx->sqe_ptr];
		/* Check if it's recycled page or else unmap DMA mapping */
		if (page && (page_ref_count(page) == 1))
		nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
		hdr->subdesc_cnt);

		/* Release page reference for recycling */
		if (page)
		put_page(page);
		sq->xdp_page[cqe_tx->sqe_ptr] = (u64)NULL;
		*subdesc_cnt += hdr->subdesc_cnt + 1;
		return;
		}

		nicvf_check_cqe_tx_errs(nic, cqe_tx);
		skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
		if (skb) {
		/* Check for dummy descriptor used for HW TSO offload on 88xx */
		@@ -528,12 +634,12 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
		(struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
		nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2,
		tso_sqe->subdesc_cnt);
		nicvf_put_sq_desc(sq, tso_sqe->subdesc_cnt + 1);
		*subdesc_cnt += tso_sqe->subdesc_cnt + 1;
		} else {
		nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
		hdr->subdesc_cnt);
		}
		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
		*subdesc_cnt += hdr->subdesc_cnt + 1;
		prefetch(skb);
		(*tx_pkts)++;
		*tx_bytes += skb->len;
		@@ -544,7 +650,7 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
		* a SKB attached, so just free SQEs here.
		*/
		if (!nic->hw_tso)
		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
		*subdesc_cnt += hdr->subdesc_cnt + 1;
		}
		}

		@@ -578,9 +684,9 @@ static inline void nicvf_set_rxhash(struct net_device *netdev,

		static void nicvf_rcv_pkt_handler(struct net_device *netdev,
		struct napi_struct *napi,
		struct cqe_rx_t *cqe_rx)
		struct cqe_rx_t cqe_rx, struct snd_queue sq)
		{
		struct sk_buff *skb;
		struct sk_buff *skb = NULL;
		struct nicvf *nic = netdev_priv(netdev);
		struct nicvf *snic = nic;
		int err = 0;
		@@ -595,16 +701,25 @@ static void nicvf_rcv_pkt_handler(struct net_device *netdev,
		}

		/* Check for errors */
		if (cqe_rx->err_level \|\| cqe_rx->err_opcode) {
		err = nicvf_check_cqe_rx_errs(nic, cqe_rx);
		if (err && !cqe_rx->rb_cnt)
		return;
		}

		skb = nicvf_get_rcv_skb(snic, cqe_rx);
		if (!skb) {
		netdev_dbg(nic->netdev, "Packet not received\n");
		/* For XDP, ignore pkts spanning multiple pages */
		if (nic->xdp_prog && (cqe_rx->rb_cnt == 1)) {
		/* Packet consumed by XDP */
		if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, &skb))
		return;
		} else {
		skb = nicvf_get_rcv_skb(snic, cqe_rx,
		nic->xdp_prog ? true : false);
		}

		if (!skb)
		return;

		if (netif_msg_pktdata(nic)) {
		netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name,
		skb, skb->len);
		@@ -646,13 +761,14 @@ static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
		{
		int processed_cqe, work_done = 0, tx_done = 0;
		int cqe_count, cqe_head;
		int subdesc_cnt = 0;
		struct nicvf *nic = netdev_priv(netdev);
		struct queue_set *qs = nic->qs;
		struct cmp_queue *cq = &qs->cq[cq_idx];
		struct cqe_rx_t *cq_desc;
		struct netdev_queue *txq;
		struct snd_queue *sq;
		unsigned int tx_pkts = 0, tx_bytes = 0;
		struct snd_queue *sq = &qs->sq[cq_idx];
		unsigned int tx_pkts = 0, tx_bytes = 0, txq_idx;

		spin_lock_bh(&cq->lock);
		loop:
		@@ -667,8 +783,6 @@ loop:
		cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
		cqe_head &= 0xFFFF;

		netdev_dbg(nic->netdev, "%s CQ%d cqe_count %d cqe_head %d\n",
		__func__, cq_idx, cqe_count, cqe_head);
		while (processed_cqe < cqe_count) {
		/* Get the CQ descriptor */
		cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
		@@ -682,17 +796,15 @@ loop:
		break;
		}

		netdev_dbg(nic->netdev, "CQ%d cq_desc->cqe_type %d\n",
		cq_idx, cq_desc->cqe_type);
		switch (cq_desc->cqe_type) {
		case CQE_TYPE_RX:
		nicvf_rcv_pkt_handler(netdev, napi, cq_desc);
		nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq);
		work_done++;
		break;
		case CQE_TYPE_SEND:
		nicvf_snd_pkt_handler(netdev,
		(void *)cq_desc, CQE_TYPE_SEND,
		budget, &tx_pkts, &tx_bytes);
		nicvf_snd_pkt_handler(netdev, (void *)cq_desc,
		budget, &subdesc_cnt,
		&tx_pkts, &tx_bytes);
		tx_done++;
		break;
		case CQE_TYPE_INVALID:
		@@ -704,9 +816,6 @@ loop:
		}
		processed_cqe++;
		}
		netdev_dbg(nic->netdev,
		"%s CQ%d processed_cqe %d work_done %d budget %d\n",
		__func__, cq_idx, processed_cqe, work_done, budget);

		/* Ring doorbell to inform H/W to reuse processed CQEs */
		nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
		@@ -716,13 +825,26 @@ loop:
		goto loop;

		done:
		/* Update SQ's descriptor free count */
		if (subdesc_cnt)
		nicvf_put_sq_desc(sq, subdesc_cnt);

		txq_idx = nicvf_netdev_qidx(nic, cq_idx);
		/* Handle XDP TX queues */
		if (nic->pnicvf->xdp_prog) {
		if (txq_idx < nic->pnicvf->xdp_tx_queues) {
		nicvf_xdp_sq_doorbell(nic, sq, cq_idx);
		goto out;
		}
		nic = nic->pnicvf;
		txq_idx -= nic->pnicvf->xdp_tx_queues;
		}

		/* Wakeup TXQ if its stopped earlier due to SQ full */
		sq = &nic->qs->sq[cq_idx];
		if (tx_done \|\|
		(atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) {
		netdev = nic->pnicvf->netdev;
		txq = netdev_get_tx_queue(netdev,
		nicvf_netdev_qidx(nic, cq_idx));
		txq = netdev_get_tx_queue(netdev, txq_idx);
		if (tx_pkts)
		netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);

		@@ -735,10 +857,11 @@ done:
		if (netif_msg_tx_err(nic))
		netdev_warn(netdev,
		"%s: Transmit queue wakeup SQ%d\n",
		netdev->name, cq_idx);
		netdev->name, txq_idx);
		}
		}

		out:
		spin_unlock_bh(&cq->lock);
		return work_done;
		}
		@@ -1054,6 +1177,13 @@ static netdev_tx_t nicvf_xmit(struct sk_buff skb, struct net_device netdev)
		return NETDEV_TX_OK;
		}

		/* In XDP case, initial HW tx queues are used for XDP,
		* but stack's queue mapping starts at '0', so skip the
		* Tx queues attached to Rx queues for XDP.
		*/
		if (nic->xdp_prog)
		qid += nic->xdp_tx_queues;

		snic = nic;
		/* Get secondary Qset's SQ structure */
		if (qid >= MAX_SND_QUEUES_PER_QS) {
		@@ -1531,6 +1661,114 @@ static int nicvf_set_features(struct net_device *netdev,
		return 0;
		}

		static void nicvf_set_xdp_queues(struct nicvf *nic, bool bpf_attached)
		{
		u8 cq_count, txq_count;

		/* Set XDP Tx queue count same as Rx queue count */
		if (!bpf_attached)
		nic->xdp_tx_queues = 0;
		else
		nic->xdp_tx_queues = nic->rx_queues;

		/* If queue count > MAX_CMP_QUEUES_PER_QS, then additional qsets
		* needs to be allocated, check how many.
		*/
		txq_count = nic->xdp_tx_queues + nic->tx_queues;
		cq_count = max(nic->rx_queues, txq_count);
		if (cq_count > MAX_CMP_QUEUES_PER_QS) {
		nic->sqs_count = roundup(cq_count, MAX_CMP_QUEUES_PER_QS);
		nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1;
		} else {
		nic->sqs_count = 0;
		}

		/* Set primary Qset's resources */
		nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS);
		nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS);
		nic->qs->cq_cnt = max_t(u8, nic->qs->rq_cnt, nic->qs->sq_cnt);

		/* Update stack */
		nicvf_set_real_num_queues(nic->netdev, nic->tx_queues, nic->rx_queues);
		}

		static int nicvf_xdp_setup(struct nicvf nic, struct bpf_prog prog)
		{
		struct net_device *dev = nic->netdev;
		bool if_up = netif_running(nic->netdev);
		struct bpf_prog *old_prog;
		bool bpf_attached = false;

		/* For now just support only the usual MTU sized frames */
		if (prog && (dev->mtu > 1500)) {
		netdev_warn(dev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
		dev->mtu);
		return -EOPNOTSUPP;
		}

		/* ALL SQs attached to CQs i.e same as RQs, are treated as
		* XDP Tx queues and more Tx queues are allocated for
		* network stack to send pkts out.
		*
		* No of Tx queues are either same as Rx queues or whatever
		* is left in max no of queues possible.
		*/
		if ((nic->rx_queues + nic->tx_queues) > nic->max_queues) {
		netdev_warn(dev,
		"Failed to attach BPF prog, RXQs + TXQs > Max %d\n",
		nic->max_queues);
		return -ENOMEM;
		}

		if (if_up)
		nicvf_stop(nic->netdev);

		old_prog = xchg(&nic->xdp_prog, prog);
		/* Detach old prog, if any */
		if (old_prog)
		bpf_prog_put(old_prog);

		if (nic->xdp_prog) {
		/* Attach BPF program */
		nic->xdp_prog = bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);
		if (!IS_ERR(nic->xdp_prog))
		bpf_attached = true;
		}

		/* Calculate Tx queues needed for XDP and network stack */
		nicvf_set_xdp_queues(nic, bpf_attached);

		if (if_up) {
		/* Reinitialize interface, clean slate */
		nicvf_open(nic->netdev);
		netif_trans_update(nic->netdev);
		}

		return 0;
		}

		static int nicvf_xdp(struct net_device netdev, struct netdev_xdp xdp)
		{
		struct nicvf *nic = netdev_priv(netdev);

		/* To avoid checks while retrieving buffer address from CQE_RX,
		* do not support XDP for T88 pass1.x silicons which are anyway
		* not in use widely.
		*/
		if (pass1_silicon(nic->pdev))
		return -EOPNOTSUPP;

		switch (xdp->command) {
		case XDP_SETUP_PROG:
		return nicvf_xdp_setup(nic, xdp->prog);
		case XDP_QUERY_PROG:
		xdp->prog_attached = !!nic->xdp_prog;
		return 0;
		default:
		return -EINVAL;
		}
		}

		static const struct net_device_ops nicvf_netdev_ops = {
		.ndo_open = nicvf_open,
		.ndo_stop = nicvf_stop,
		@@ -1541,6 +1779,7 @@ static const struct net_device_ops nicvf_netdev_ops = {
		.ndo_tx_timeout = nicvf_tx_timeout,
		.ndo_fix_features = nicvf_fix_features,
		.ndo_set_features = nicvf_set_features,
		.ndo_xdp = nicvf_xdp,
		};

		static int nicvf_probe(struct pci_dev pdev, const struct pci_device_id ent)

drivers/net/ethernet/cavium/thunder/nicvf_queues.c

+319 −68

File changed.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/cavium/thunder/nicvf_queues.h

+31 −1

Original line number	Diff line number	Diff line
		@@ -10,6 +10,7 @@
		#define NICVF_QUEUES_H

		#include <linux/netdevice.h>
		#include <linux/iommu.h>
		#include "q_struct.h"

		#define MAX_QUEUE_SET 128
		@@ -213,6 +214,12 @@ struct q_desc_mem {
		void *unalign_base;
		};

		struct pgcache {
		struct page *page;
		int ref_count;
		u64 dma_addr;
		};

		struct rbdr {
		bool enable;
		u32 dma_size;
		@@ -222,6 +229,13 @@ struct rbdr {
		u32 head;
		u32 tail;
		struct q_desc_mem dmem;
		bool is_xdp;

		/* For page recycling */
		int pgidx;
		int pgcnt;
		int pgalloc;
		struct pgcache *pgcache;
		} ____cacheline_aligned_in_smp;

		struct rcv_queue {
		@@ -258,6 +272,10 @@ struct snd_queue {
		u32 tail;
		u64 *skbuff;
		void *desc;
		u64 *xdp_page;
		u16 xdp_desc_cnt;
		u16 xdp_free_cnt;
		bool is_xdp;

		#define TSO_HEADER_SIZE 128
		/* For TSO segment's header */
		@@ -301,6 +319,14 @@ struct queue_set {

		#define CQ_ERR_MASK (CQ_WR_FULL \| CQ_WR_DISABLE \| CQ_WR_FAULT)

		static inline u64 nicvf_iova_to_phys(struct nicvf *nic, dma_addr_t dma_addr)
		{
		/* Translation is installed only when IOMMU is present */
		if (nic->iommu_domain)
		return iommu_iova_to_phys(nic->iommu_domain, dma_addr);
		return dma_addr;
		}

		void nicvf_unmap_sndq_buffers(struct nicvf nic, struct snd_queue sq,
		int hdr_sqe, u8 subdesc_cnt);
		void nicvf_config_vlan_stripping(struct nicvf *nic,
		@@ -318,8 +344,12 @@ void nicvf_sq_free_used_descs(struct net_device *netdev,
		struct snd_queue *sq, int qidx);
		int nicvf_sq_append_skb(struct nicvf nic, struct snd_queue sq,
		struct sk_buff *skb, u8 sq_num);
		int nicvf_xdp_sq_append_pkt(struct nicvf nic, struct snd_queue sq,
		u64 bufaddr, u64 dma_addr, u16 len);
		void nicvf_xdp_sq_doorbell(struct nicvf nic, struct snd_queue sq, int sq_num);

		struct sk_buff nicvf_get_rcv_skb(struct nicvf nic, struct cqe_rx_t *cqe_rx);
		struct sk_buff nicvf_get_rcv_skb(struct nicvf nic,
		struct cqe_rx_t *cqe_rx, bool xdp);
		void nicvf_rbdr_task(unsigned long data);
		void nicvf_rbdr_work(struct work_struct *work);

Admin message