Merge branch 'ibmvnic-performance-improvements-and-other-updates' (16de5970) · Commits · 戴 / test

drivers/net/ethernet/ibm/ibmvnic.c

+242 −156

Original line number	Diff line number	Diff line
		@@ -84,8 +84,6 @@ static int ibmvnic_reset_crq(struct ibmvnic_adapter *);
		static int ibmvnic_send_crq_init(struct ibmvnic_adapter *);
		static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *);
		static int ibmvnic_send_crq(struct ibmvnic_adapter , union ibmvnic_crq );
		static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
		union sub_crq *sub_crq);
		static int send_subcrq_indirect(struct ibmvnic_adapter *, u64, u64, u64);
		static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance);
		static int enable_scrq_irq(struct ibmvnic_adapter *,
		@@ -306,9 +304,11 @@ static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
		int count = pool->size - atomic_read(&pool->available);
		u64 handle = adapter->rx_scrq[pool->index]->handle;
		struct device *dev = &adapter->vdev->dev;
		struct ibmvnic_ind_xmit_queue *ind_bufp;
		struct ibmvnic_sub_crq_queue *rx_scrq;
		union sub_crq *sub_crq;
		int buffers_added = 0;
		unsigned long lpar_rc;
		union sub_crq sub_crq;
		struct sk_buff *skb;
		unsigned int offset;
		dma_addr_t dma_addr;
		@@ -320,8 +320,10 @@ static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
		if (!pool->active)
		return;

		rx_scrq = adapter->rx_scrq[pool->index];
		ind_bufp = &rx_scrq->ind_buf;
		for (i = 0; i < count; ++i) {
		skb = alloc_skb(pool->buff_size, GFP_ATOMIC);
		skb = netdev_alloc_skb(adapter->netdev, pool->buff_size);
		if (!skb) {
		dev_err(dev, "Couldn't replenish rx buff\n");
		adapter->replenish_no_mem++;
		@@ -346,12 +348,13 @@ static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
		pool->rx_buff[index].pool_index = pool->index;
		pool->rx_buff[index].size = pool->buff_size;

		memset(&sub_crq, 0, sizeof(sub_crq));
		sub_crq.rx_add.first = IBMVNIC_CRQ_CMD;
		sub_crq.rx_add.correlator =
		sub_crq = &ind_bufp->indir_arr[ind_bufp->index++];
		memset(sub_crq, 0, sizeof(*sub_crq));
		sub_crq->rx_add.first = IBMVNIC_CRQ_CMD;
		sub_crq->rx_add.correlator =
		cpu_to_be64((u64)&pool->rx_buff[index]);
		sub_crq.rx_add.ioba = cpu_to_be32(dma_addr);
		sub_crq.rx_add.map_id = pool->long_term_buff.map_id;
		sub_crq->rx_add.ioba = cpu_to_be32(dma_addr);
		sub_crq->rx_add.map_id = pool->long_term_buff.map_id;

		/* The length field of the sCRQ is defined to be 24 bits so the
		* buffer size needs to be left shifted by a byte before it is
		@@ -361,15 +364,20 @@ static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
		#ifdef __LITTLE_ENDIAN__
		shift = 8;
		#endif
		sub_crq.rx_add.len = cpu_to_be32(pool->buff_size << shift);

		lpar_rc = send_subcrq(adapter, handle, &sub_crq);
		sub_crq->rx_add.len = cpu_to_be32(pool->buff_size << shift);
		pool->next_free = (pool->next_free + 1) % pool->size;
		if (ind_bufp->index == IBMVNIC_MAX_IND_DESCS \|\|
		i == count - 1) {
		lpar_rc =
		send_subcrq_indirect(adapter, handle,
		(u64)ind_bufp->indir_dma,
		(u64)ind_bufp->index);
		if (lpar_rc != H_SUCCESS)
		goto failure;

		buffers_added++;
		adapter->replenish_add_buff_success++;
		pool->next_free = (pool->next_free + 1) % pool->size;
		buffers_added += ind_bufp->index;
		adapter->replenish_add_buff_success += ind_bufp->index;
		ind_bufp->index = 0;
		}
		}
		atomic_add(buffers_added, &pool->available);
		return;
		@@ -377,13 +385,20 @@ static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
		failure:
		if (lpar_rc != H_PARAMETER && lpar_rc != H_CLOSED)
		dev_err_ratelimited(dev, "rx: replenish packet buffer failed\n");
		for (i = ind_bufp->index - 1; i >= 0; --i) {
		struct ibmvnic_rx_buff *rx_buff;

		pool->next_free = pool->next_free == 0 ?
		pool->size - 1 : pool->next_free - 1;
		sub_crq = &ind_bufp->indir_arr[i];
		rx_buff = (struct ibmvnic_rx_buff *)
		be64_to_cpu(sub_crq->rx_add.correlator);
		index = (int)(rx_buff - pool->rx_buff);
		pool->free_map[pool->next_free] = index;
		dev_kfree_skb_any(pool->rx_buff[index].skb);
		pool->rx_buff[index].skb = NULL;

		dev_kfree_skb_any(skb);
		adapter->replenish_add_buff_failure++;
		atomic_add(buffers_added, &pool->available);

		}
		ind_bufp->index = 0;
		if (lpar_rc == H_CLOSED \|\| adapter->failover_pending) {
		/* Disable buffer pool replenishment and report carrier off if
		* queue is closed or pending failover.
		@@ -483,7 +498,7 @@ static int reset_rx_pools(struct ibmvnic_adapter *adapter)

		if (rx_pool->buff_size != buff_size) {
		free_long_term_buff(adapter, &rx_pool->long_term_buff);
		rx_pool->buff_size = buff_size;
		rx_pool->buff_size = ALIGN(buff_size, L1_CACHE_BYTES);
		rc = alloc_long_term_buff(adapter,
		&rx_pool->long_term_buff,
		rx_pool->size *
		@@ -577,7 +592,7 @@ static int init_rx_pools(struct net_device *netdev)

		rx_pool->size = adapter->req_rx_add_entries_per_subcrq;
		rx_pool->index = i;
		rx_pool->buff_size = buff_size;
		rx_pool->buff_size = ALIGN(buff_size, L1_CACHE_BYTES);
		rx_pool->active = 1;

		rx_pool->free_map = kcalloc(rx_pool->size, sizeof(int),
		@@ -730,6 +745,7 @@ static int init_tx_pools(struct net_device *netdev)
		{
		struct ibmvnic_adapter *adapter = netdev_priv(netdev);
		int tx_subcrqs;
		u64 buff_size;
		int i, rc;

		tx_subcrqs = adapter->num_active_tx_scrqs;
		@@ -746,9 +762,11 @@ static int init_tx_pools(struct net_device *netdev)
		adapter->num_active_tx_pools = tx_subcrqs;

		for (i = 0; i < tx_subcrqs; i++) {
		buff_size = adapter->req_mtu + VLAN_HLEN;
		buff_size = ALIGN(buff_size, L1_CACHE_BYTES);
		rc = init_one_tx_pool(netdev, &adapter->tx_pool[i],
		adapter->req_tx_entries_per_subcrq,
		adapter->req_mtu + VLAN_HLEN);
		buff_size);
		if (rc) {
		release_tx_pools(adapter);
		return rc;
		@@ -1148,6 +1166,7 @@ static int __ibmvnic_open(struct net_device *netdev)
		if (prev_state == VNIC_CLOSED)
		enable_irq(adapter->tx_scrq[i]->irq);
		enable_scrq_irq(adapter, adapter->tx_scrq[i]);
		netdev_tx_reset_queue(netdev_get_tx_queue(netdev, i));
		}

		rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);
		@@ -1478,17 +1497,18 @@ static int create_hdr_descs(u8 hdr_field, u8 hdr_data, int len, int hdr_len,
		* L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.
		*/

		static void build_hdr_descs_arr(struct ibmvnic_tx_buff *txbuff,
		static void build_hdr_descs_arr(struct sk_buff *skb,
		union sub_crq *indir_arr,
		int *num_entries, u8 hdr_field)
		{
		int hdr_len[3] = {0, 0, 0};
		u8 hdr_data[140] = {0};
		int tot_len;
		u8 *hdr_data = txbuff->hdr_data;

		tot_len = build_hdr_data(hdr_field, txbuff->skb, hdr_len,
		txbuff->hdr_data);
		tot_len = build_hdr_data(hdr_field, skb, hdr_len,
		hdr_data);
		*num_entries += create_hdr_descs(hdr_field, hdr_data, tot_len, hdr_len,
		txbuff->indir_arr + 1);
		indir_arr + 1);
		}

		static int ibmvnic_xmit_workarounds(struct sk_buff *skb,
		@@ -1506,17 +1526,95 @@ static int ibmvnic_xmit_workarounds(struct sk_buff *skb,
		return 0;
		}

		static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,
		struct ibmvnic_sub_crq_queue *tx_scrq)
		{
		struct ibmvnic_ind_xmit_queue *ind_bufp;
		struct ibmvnic_tx_buff *tx_buff;
		struct ibmvnic_tx_pool *tx_pool;
		union sub_crq tx_scrq_entry;
		int queue_num;
		int entries;
		int index;
		int i;

		ind_bufp = &tx_scrq->ind_buf;
		entries = (u64)ind_bufp->index;
		queue_num = tx_scrq->pool_index;

		for (i = entries - 1; i >= 0; --i) {
		tx_scrq_entry = ind_bufp->indir_arr[i];
		if (tx_scrq_entry.v1.type != IBMVNIC_TX_DESC)
		continue;
		index = be32_to_cpu(tx_scrq_entry.v1.correlator);
		if (index & IBMVNIC_TSO_POOL_MASK) {
		tx_pool = &adapter->tso_pool[queue_num];
		index &= ~IBMVNIC_TSO_POOL_MASK;
		} else {
		tx_pool = &adapter->tx_pool[queue_num];
		}
		tx_pool->free_map[tx_pool->consumer_index] = index;
		tx_pool->consumer_index = tx_pool->consumer_index == 0 ?
		tx_pool->num_buffers - 1 :
		tx_pool->consumer_index - 1;
		tx_buff = &tx_pool->tx_buff[index];
		adapter->netdev->stats.tx_packets--;
		adapter->netdev->stats.tx_bytes -= tx_buff->skb->len;
		adapter->tx_stats_buffers[queue_num].packets--;
		adapter->tx_stats_buffers[queue_num].bytes -=
		tx_buff->skb->len;
		dev_kfree_skb_any(tx_buff->skb);
		tx_buff->skb = NULL;
		adapter->netdev->stats.tx_dropped++;
		}
		ind_bufp->index = 0;
		if (atomic_sub_return(entries, &tx_scrq->used) <=
		(adapter->req_tx_entries_per_subcrq / 2) &&
		__netif_subqueue_stopped(adapter->netdev, queue_num)) {
		netif_wake_subqueue(adapter->netdev, queue_num);
		netdev_dbg(adapter->netdev, "Started queue %d\n",
		queue_num);
		}
		}

		static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter *adapter,
		struct ibmvnic_sub_crq_queue *tx_scrq)
		{
		struct ibmvnic_ind_xmit_queue *ind_bufp;
		u64 dma_addr;
		u64 entries;
		u64 handle;
		int rc;

		ind_bufp = &tx_scrq->ind_buf;
		dma_addr = (u64)ind_bufp->indir_dma;
		entries = (u64)ind_bufp->index;
		handle = tx_scrq->handle;

		if (!entries)
		return 0;
		rc = send_subcrq_indirect(adapter, handle, dma_addr, entries);
		if (rc)
		ibmvnic_tx_scrq_clean_buffer(adapter, tx_scrq);
		else
		ind_bufp->index = 0;
		return 0;
		}

		static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		{
		struct ibmvnic_adapter *adapter = netdev_priv(netdev);
		int queue_num = skb_get_queue_mapping(skb);
		u8 hdrs = (u8 )&adapter->tx_rx_desc_req;
		struct device *dev = &adapter->vdev->dev;
		struct ibmvnic_ind_xmit_queue *ind_bufp;
		struct ibmvnic_tx_buff *tx_buff = NULL;
		struct ibmvnic_sub_crq_queue *tx_scrq;
		struct ibmvnic_tx_pool *tx_pool;
		unsigned int tx_send_failed = 0;
		netdev_tx_t ret = NETDEV_TX_OK;
		unsigned int tx_map_failed = 0;
		union sub_crq indir_arr[16];
		unsigned int tx_dropped = 0;
		unsigned int tx_packets = 0;
		unsigned int tx_bytes = 0;
		@@ -1529,8 +1627,10 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		unsigned char *dst;
		int index = 0;
		u8 proto = 0;
		u64 handle;
		netdev_tx_t ret = NETDEV_TX_OK;

		tx_scrq = adapter->tx_scrq[queue_num];
		txq = netdev_get_tx_queue(netdev, queue_num);
		ind_bufp = &tx_scrq->ind_buf;

		if (test_bit(0, &adapter->resetting)) {
		if (!netif_subqueue_stopped(netdev, skb))
		@@ -1540,6 +1640,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		tx_send_failed++;
		tx_dropped++;
		ret = NETDEV_TX_OK;
		ibmvnic_tx_scrq_flush(adapter, tx_scrq);
		goto out;
		}

		@@ -1547,6 +1648,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		tx_dropped++;
		tx_send_failed++;
		ret = NETDEV_TX_OK;
		ibmvnic_tx_scrq_flush(adapter, tx_scrq);
		goto out;
		}
		if (skb_is_gso(skb))
		@@ -1554,10 +1656,6 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		else
		tx_pool = &adapter->tx_pool[queue_num];

		tx_scrq = adapter->tx_scrq[queue_num];
		txq = netdev_get_tx_queue(netdev, skb_get_queue_mapping(skb));
		handle = tx_scrq->handle;

		index = tx_pool->free_map[tx_pool->consumer_index];

		if (index == IBMVNIC_INVALID_MAP) {
		@@ -1565,6 +1663,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		tx_send_failed++;
		tx_dropped++;
		ret = NETDEV_TX_OK;
		ibmvnic_tx_scrq_flush(adapter, tx_scrq);
		goto out;
		}

		@@ -1600,11 +1699,8 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)

		tx_buff = &tx_pool->tx_buff[index];
		tx_buff->skb = skb;
		tx_buff->data_dma[0] = data_dma_addr;
		tx_buff->data_len[0] = skb->len;
		tx_buff->index = index;
		tx_buff->pool_index = queue_num;
		tx_buff->last_frag = true;

		memset(&tx_crq, 0, sizeof(tx_crq));
		tx_crq.v1.first = IBMVNIC_CRQ_CMD;
		@@ -1649,55 +1745,29 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		tx_crq.v1.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
		hdrs += 2;
		}
		/* determine if l2/3/4 headers are sent to firmware */
		if ((*hdrs >> 7) & 1) {
		build_hdr_descs_arr(tx_buff, &num_entries, *hdrs);

		if ((*hdrs >> 7) & 1)
		build_hdr_descs_arr(skb, indir_arr, &num_entries, *hdrs);

		tx_crq.v1.n_crq_elem = num_entries;
		tx_buff->num_entries = num_entries;
		tx_buff->indir_arr[0] = tx_crq;
		tx_buff->indir_dma = dma_map_single(dev, tx_buff->indir_arr,
		sizeof(tx_buff->indir_arr),
		DMA_TO_DEVICE);
		if (dma_mapping_error(dev, tx_buff->indir_dma)) {
		dev_kfree_skb_any(skb);
		tx_buff->skb = NULL;
		if (!firmware_has_feature(FW_FEATURE_CMO))
		dev_err(dev, "tx: unable to map descriptor array\n");
		tx_map_failed++;
		tx_dropped++;
		ret = NETDEV_TX_OK;
		goto tx_err_out;
		}
		lpar_rc = send_subcrq_indirect(adapter, handle,
		(u64)tx_buff->indir_dma,
		(u64)num_entries);
		dma_unmap_single(dev, tx_buff->indir_dma,
		sizeof(tx_buff->indir_arr), DMA_TO_DEVICE);
		} else {
		tx_buff->num_entries = num_entries;
		lpar_rc = send_subcrq(adapter, handle,
		&tx_crq);
		}
		if (lpar_rc != H_SUCCESS) {
		if (lpar_rc != H_CLOSED && lpar_rc != H_PARAMETER)
		dev_err_ratelimited(dev, "tx: send failed\n");
		dev_kfree_skb_any(skb);
		tx_buff->skb = NULL;

		if (lpar_rc == H_CLOSED \|\| adapter->failover_pending) {
		/* Disable TX and report carrier off if queue is closed
		* or pending failover.
		* Firmware guarantees that a signal will be sent to the
		* driver, triggering a reset or some other action.
		*/
		netif_tx_stop_all_queues(netdev);
		netif_carrier_off(netdev);
		/* flush buffer if current entry can not fit */
		if (num_entries + ind_bufp->index > IBMVNIC_MAX_IND_DESCS) {
		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq);
		if (lpar_rc != H_SUCCESS)
		goto tx_flush_err;
		}

		tx_send_failed++;
		tx_dropped++;
		ret = NETDEV_TX_OK;
		goto tx_err_out;
		indir_arr[0] = tx_crq;
		memcpy(&ind_bufp->indir_arr[ind_bufp->index], &indir_arr[0],
		num_entries * sizeof(struct ibmvnic_generic_scrq));
		ind_bufp->index += num_entries;
		if (__netdev_tx_sent_queue(txq, skb->len,
		netdev_xmit_more() &&
		ind_bufp->index < IBMVNIC_MAX_IND_DESCS)) {
		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq);
		if (lpar_rc != H_SUCCESS)
		goto tx_err;
		}

		if (atomic_add_return(num_entries, &tx_scrq->used)
		@@ -1712,14 +1782,26 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff skb, struct net_device netdev)
		ret = NETDEV_TX_OK;
		goto out;

		tx_err_out:
		/* roll back consumer index and map array*/
		if (tx_pool->consumer_index == 0)
		tx_pool->consumer_index =
		tx_pool->num_buffers - 1;
		else
		tx_pool->consumer_index--;
		tx_pool->free_map[tx_pool->consumer_index] = index;
		tx_flush_err:
		dev_kfree_skb_any(skb);
		tx_buff->skb = NULL;
		tx_pool->consumer_index = tx_pool->consumer_index == 0 ?
		tx_pool->num_buffers - 1 :
		tx_pool->consumer_index - 1;
		tx_dropped++;
		tx_err:
		if (lpar_rc != H_CLOSED && lpar_rc != H_PARAMETER)
		dev_err_ratelimited(dev, "tx: send failed\n");

		if (lpar_rc == H_CLOSED \|\| adapter->failover_pending) {
		/* Disable TX and report carrier off if queue is closed
		* or pending failover.
		* Firmware guarantees that a signal will be sent to the
		* driver, triggering a reset or some other action.
		*/
		netif_tx_stop_all_queues(netdev);
		netif_carrier_off(netdev);
		}
		out:
		netdev->stats.tx_dropped += tx_dropped;
		netdev->stats.tx_bytes += tx_bytes;
		@@ -2368,10 +2450,17 @@ static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,

		static int ibmvnic_poll(struct napi_struct *napi, int budget)
		{
		struct net_device *netdev = napi->dev;
		struct ibmvnic_adapter *adapter = netdev_priv(netdev);
		int scrq_num = (int)(napi - adapter->napi);
		int frames_processed = 0;
		struct ibmvnic_sub_crq_queue *rx_scrq;
		struct ibmvnic_adapter *adapter;
		struct net_device *netdev;
		int frames_processed;
		int scrq_num;

		netdev = napi->dev;
		adapter = netdev_priv(netdev);
		scrq_num = (int)(napi - adapter->napi);
		frames_processed = 0;
		rx_scrq = adapter->rx_scrq[scrq_num];

		restart_poll:
		while (frames_processed < budget) {
		@@ -2384,14 +2473,14 @@ restart_poll:

		if (unlikely(test_bit(0, &adapter->resetting) &&
		adapter->reset_reason != VNIC_RESET_NON_FATAL)) {
		enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
		enable_scrq_irq(adapter, rx_scrq);
		napi_complete_done(napi, frames_processed);
		return frames_processed;
		}

		if (!pending_scrq(adapter, adapter->rx_scrq[scrq_num]))
		if (!pending_scrq(adapter, rx_scrq))
		break;
		next = ibmvnic_next_scrq(adapter, adapter->rx_scrq[scrq_num]);
		next = ibmvnic_next_scrq(adapter, rx_scrq);
		rx_buff =
		(struct ibmvnic_rx_buff *)be64_to_cpu(next->
		rx_comp.correlator);
		@@ -2448,18 +2537,23 @@ restart_poll:
		frames_processed++;
		}

		if (adapter->state != VNIC_CLOSING)
		if (adapter->state != VNIC_CLOSING &&
		((atomic_read(&adapter->rx_pool[scrq_num].available) <
		adapter->req_rx_add_entries_per_subcrq / 2) \|\|
		frames_processed < budget))
		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);

		if (frames_processed < budget) {
		enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
		napi_complete_done(napi, frames_processed);
		if (pending_scrq(adapter, adapter->rx_scrq[scrq_num]) &&
		napi_reschedule(napi)) {
		disable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
		if (napi_complete_done(napi, frames_processed)) {
		enable_scrq_irq(adapter, rx_scrq);
		if (pending_scrq(adapter, rx_scrq)) {
		rmb();
		if (napi_reschedule(napi)) {
		disable_scrq_irq(adapter, rx_scrq);
		goto restart_poll;
		}
		}
		}
		}
		return frames_processed;
		}

		@@ -2858,6 +2952,7 @@ static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,
		memset(scrq->msgs, 0, 4 * PAGE_SIZE);
		atomic_set(&scrq->used, 0);
		scrq->cur = 0;
		scrq->ind_buf.index = 0;

		rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
		4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
		@@ -2909,6 +3004,11 @@ static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,
		}
		}

		dma_free_coherent(dev,
		IBMVNIC_IND_ARR_SZ,
		scrq->ind_buf.indir_arr,
		scrq->ind_buf.indir_dma);

		dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
		DMA_BIDIRECTIONAL);
		free_pages((unsigned long)scrq->msgs, 2);
		@@ -2955,6 +3055,17 @@ static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter

		scrq->adapter = adapter;
		scrq->size = 4 * PAGE_SIZE / sizeof(*scrq->msgs);
		scrq->ind_buf.index = 0;

		scrq->ind_buf.indir_arr =
		dma_alloc_coherent(dev,
		IBMVNIC_IND_ARR_SZ,
		&scrq->ind_buf.indir_dma,
		GFP_KERNEL);

		if (!scrq->ind_buf.indir_arr)
		goto indir_failed;

		spin_lock_init(&scrq->lock);

		netdev_dbg(adapter->netdev,
		@@ -2963,6 +3074,12 @@ static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter

		return scrq;

		indir_failed:
		do {
		rc = plpar_hcall_norets(H_FREE_SUB_CRQ,
		adapter->vdev->unit_address,
		scrq->crq_num);
		} while (rc == H_BUSY \|\| rc == H_IS_LONG_BUSY(rc));
		reg_failed:
		dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
		DMA_BIDIRECTIONAL);
		@@ -3077,14 +3194,17 @@ static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,
		struct device *dev = &adapter->vdev->dev;
		struct ibmvnic_tx_pool *tx_pool;
		struct ibmvnic_tx_buff *txbuff;
		struct netdev_queue *txq;
		union sub_crq *next;
		int index;
		int i, j;
		int i;

		restart_loop:
		while (pending_scrq(adapter, scrq)) {
		unsigned int pool = scrq->pool_index;
		int num_entries = 0;
		int total_bytes = 0;
		int num_packets = 0;

		next = ibmvnic_next_scrq(adapter, scrq);
		for (i = 0; i < next->tx_comp.num_comps; i++) {
		@@ -3102,21 +3222,16 @@ restart_loop:
		}

		txbuff = &tx_pool->tx_buff[index];

		for (j = 0; j < IBMVNIC_MAX_FRAGS_PER_CRQ; j++) {
		if (!txbuff->data_dma[j])
		continue;

		txbuff->data_dma[j] = 0;
		}

		if (txbuff->last_frag) {
		dev_kfree_skb_any(txbuff->skb);
		num_packets++;
		num_entries += txbuff->num_entries;
		if (txbuff->skb) {
		total_bytes += txbuff->skb->len;
		dev_consume_skb_irq(txbuff->skb);
		txbuff->skb = NULL;
		} else {
		netdev_warn(adapter->netdev,
		"TX completion received with NULL socket buffer\n");
		}

		num_entries += txbuff->num_entries;

		tx_pool->free_map[tx_pool->producer_index] = index;
		tx_pool->producer_index =
		(tx_pool->producer_index + 1) %
		@@ -3125,6 +3240,9 @@ restart_loop:
		/* remove tx_comp scrq*/
		next->tx_comp.first = 0;

		txq = netdev_get_tx_queue(adapter->netdev, scrq->pool_index);
		netdev_tx_completed_queue(txq, num_packets, total_bytes);

		if (atomic_sub_return(num_entries, &scrq->used) <=
		(adapter->req_tx_entries_per_subcrq / 2) &&
		__netif_subqueue_stopped(adapter->netdev,
		@@ -3524,38 +3642,6 @@ static void print_subcrq_error(struct device dev, int rc, const char func)
		}
		}

		static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
		union sub_crq *sub_crq)
		{
		unsigned int ua = adapter->vdev->unit_address;
		struct device *dev = &adapter->vdev->dev;
		u64 u64_crq = (u64 )sub_crq;
		int rc;

		netdev_dbg(adapter->netdev,
		"Sending sCRQ %016lx: %016lx %016lx %016lx %016lx\n",
		(unsigned long int)cpu_to_be64(remote_handle),
		(unsigned long int)cpu_to_be64(u64_crq[0]),
		(unsigned long int)cpu_to_be64(u64_crq[1]),
		(unsigned long int)cpu_to_be64(u64_crq[2]),
		(unsigned long int)cpu_to_be64(u64_crq[3]));

		/* Make sure the hypervisor sees the complete request */
		mb();

		rc = plpar_hcall_norets(H_SEND_SUB_CRQ, ua,
		cpu_to_be64(remote_handle),
		cpu_to_be64(u64_crq[0]),
		cpu_to_be64(u64_crq[1]),
		cpu_to_be64(u64_crq[2]),
		cpu_to_be64(u64_crq[3]));

		if (rc)
		print_subcrq_error(dev, rc, __func__);

		return rc;
		}

		static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,
		u64 remote_handle, u64 ioba, u64 num_entries)
		{

drivers/net/ethernet/ibm/ibmvnic.h

+14 −13

Original line number	Diff line number	Diff line
		@@ -31,6 +31,8 @@
		#define IBMVNIC_BUFFS_PER_POOL 100
		#define IBMVNIC_MAX_QUEUES 16
		#define IBMVNIC_MAX_QUEUE_SZ 4096
		#define IBMVNIC_MAX_IND_DESCS 128
		#define IBMVNIC_IND_ARR_SZ (IBMVNIC_MAX_IND_DESCS * 32)

		#define IBMVNIC_TSO_BUF_SZ 65536
		#define IBMVNIC_TSO_BUFS 64
		@@ -224,8 +226,6 @@ struct ibmvnic_tx_comp_desc {
		#define IBMVNIC_TCP_CHKSUM 0x20
		#define IBMVNIC_UDP_CHKSUM 0x08

		#define IBMVNIC_MAX_FRAGS_PER_CRQ 3

		struct ibmvnic_tx_desc {
		u8 first;
		u8 type;
		@@ -861,6 +861,12 @@ union sub_crq {
		struct ibmvnic_rx_buff_add_desc rx_add;
		};

		struct ibmvnic_ind_xmit_queue {
		union sub_crq *indir_arr;
		dma_addr_t indir_dma;
		int index;
		};

		struct ibmvnic_sub_crq_queue {
		union sub_crq *msgs;
		int size, cur;
		@@ -873,10 +879,11 @@ struct ibmvnic_sub_crq_queue {
		spinlock_t lock;
		struct sk_buff *rx_skb_top;
		struct ibmvnic_adapter *adapter;
		struct ibmvnic_ind_xmit_queue ind_buf;
		atomic_t used;
		char name[32];
		u64 handle;
		};
		} ____cacheline_aligned;

		struct ibmvnic_long_term_buff {
		unsigned char *buff;
		@@ -887,14 +894,8 @@ struct ibmvnic_long_term_buff {

		struct ibmvnic_tx_buff {
		struct sk_buff *skb;
		dma_addr_t data_dma[IBMVNIC_MAX_FRAGS_PER_CRQ];
		unsigned int data_len[IBMVNIC_MAX_FRAGS_PER_CRQ];
		int index;
		int pool_index;
		bool last_frag;
		union sub_crq indir_arr[6];
		u8 hdr_data[140];
		dma_addr_t indir_dma;
		int num_entries;
		};

		@@ -906,7 +907,7 @@ struct ibmvnic_tx_pool {
		struct ibmvnic_long_term_buff long_term_buff;
		int num_buffers;
		int buf_size;
		};
		} ____cacheline_aligned;

		struct ibmvnic_rx_buff {
		struct sk_buff *skb;
		@@ -927,7 +928,7 @@ struct ibmvnic_rx_pool {
		int next_alloc;
		int active;
		struct ibmvnic_long_term_buff long_term_buff;
		};
		} ____cacheline_aligned;

		struct ibmvnic_vpd {
		unsigned char *buff;
		@@ -1014,8 +1015,8 @@ struct ibmvnic_adapter {
		atomic_t running_cap_crqs;
		bool wait_capability;

		struct ibmvnic_sub_crq_queue **tx_scrq;
		struct ibmvnic_sub_crq_queue **rx_scrq;
		struct ibmvnic_sub_crq_queue **tx_scrq ____cacheline_aligned;
		struct ibmvnic_sub_crq_queue **rx_scrq ____cacheline_aligned;

		/* rx structs */
		struct napi_struct *napi;

Admin message