Commit 6c3c05b0 authored by Chaitanya Kulkarni's avatar Chaitanya Kulkarni Committed by Christoph Hellwig
Browse files

nvme-core: replace ctrl page size with a macro 



Saving the nvme controller's page size was from a time when the driver
tried to use different sized pages, but this value is always set to
a constant, and has been this way for some time. Remove the 'page_size'
field and replace its usage with the constant value.

This also lets the compiler make some micro-optimizations in the io
path, and that's always a good thing.

Signed-off-by: default avatarChaitanya Kulkarni <chaitanya.kulkarni@wdc.com>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
parent 5887450b

drivers/nvme/host/core.c

+6 −13
Original line number Diff line number Diff line
@@ -2345,12 +2345,7 @@ EXPORT_SYMBOL_GPL(nvme_disable_ctrl); 


int nvme_enable_ctrl(struct nvme_ctrl *ctrl)

{

	/*

	 * Default to a 4K page size, with the intention to update this

	 * path in the future to accomodate architectures with differing

	 * kernel and IO page sizes.

	 */

	unsigned dev_page_min, page_shift = 12;

	unsigned dev_page_min;

	int ret;



	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
@@ -2360,20 +2355,18 @@ int nvme_enable_ctrl(struct nvme_ctrl *ctrl) 
	}

	dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;



	if (page_shift < dev_page_min) {

	if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {

		dev_err(ctrl->device,

			"Minimum device page size %u too large for host (%u)\n",

			1 << dev_page_min, 1 << page_shift);

			1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);

		return -ENODEV;

	}



	ctrl->page_size = 1 << page_shift;



	if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)

		ctrl->ctrl_config = NVME_CC_CSS_CSI;

	else

		ctrl->ctrl_config = NVME_CC_CSS_NVM;

	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;

	ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;

	ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;

	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;

	ctrl->ctrl_config |= NVME_CC_ENABLE;
@@ -2423,13 +2416,13 @@ static void nvme_set_queue_limits(struct nvme_ctrl *ctrl, 


	if (ctrl->max_hw_sectors) {

		u32 max_segments =

			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;

			(ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;



		max_segments = min_not_zero(max_segments, ctrl->max_segments);

		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);

		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));

	}

	blk_queue_virt_boundary(q, ctrl->page_size - 1);

	blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);

	blk_queue_dma_alignment(q, 7);

	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)

		vwc = true;

drivers/nvme/host/nvme.h

+8 −1
Original line number Diff line number Diff line
@@ -37,6 +37,14 @@ extern unsigned int admin_timeout; 
#define  NVME_INLINE_METADATA_SG_CNT  1

#endif



/*

 * Default to a 4K page size, with the intention to update this

 * path in the future to accommodate architectures with differing

 * kernel and IO page sizes.

 */

#define NVME_CTRL_PAGE_SHIFT	12

#define NVME_CTRL_PAGE_SIZE	(1 << NVME_CTRL_PAGE_SHIFT)



extern struct workqueue_struct *nvme_wq;

extern struct workqueue_struct *nvme_reset_wq;

extern struct workqueue_struct *nvme_delete_wq;
@@ -234,7 +242,6 @@ struct nvme_ctrl { 
	u32 queue_count;



	u64 cap;

	u32 page_size;

	u32 max_hw_sectors;

	u32 max_segments;

	u32 max_integrity_segments;

drivers/nvme/host/pci.c

+23 −24
Original line number Diff line number Diff line
@@ -348,8 +348,8 @@ static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db, 
 */

static int nvme_npages(unsigned size, struct nvme_dev *dev)

{

	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,

				      dev->ctrl.page_size);

	unsigned nprps = DIV_ROUND_UP(size + NVME_CTRL_PAGE_SIZE,

				      NVME_CTRL_PAGE_SIZE);

	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);

}
@@ -515,7 +515,7 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req) 
static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)

{

	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);

	const int last_prp = dev->ctrl.page_size / sizeof(__le64) - 1;

	const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;

	dma_addr_t dma_addr = iod->first_dma, next_dma_addr;

	int i;
@@ -582,34 +582,33 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, 
	struct scatterlist *sg = iod->sg;

	int dma_len = sg_dma_len(sg);

	u64 dma_addr = sg_dma_address(sg);

	u32 page_size = dev->ctrl.page_size;

	int offset = dma_addr & (page_size - 1);

	int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);

	__le64 *prp_list;

	void **list = nvme_pci_iod_list(req);

	dma_addr_t prp_dma;

	int nprps, i;



	length -= (page_size - offset);

	length -= (NVME_CTRL_PAGE_SIZE - offset);

	if (length <= 0) {

		iod->first_dma = 0;

		goto done;

	}



	dma_len -= (page_size - offset);

	dma_len -= (NVME_CTRL_PAGE_SIZE - offset);

	if (dma_len) {

		dma_addr += (page_size - offset);

		dma_addr += (NVME_CTRL_PAGE_SIZE - offset);

	} else {

		sg = sg_next(sg);

		dma_addr = sg_dma_address(sg);

		dma_len = sg_dma_len(sg);

	}



	if (length <= page_size) {

	if (length <= NVME_CTRL_PAGE_SIZE) {

		iod->first_dma = dma_addr;

		goto done;

	}



	nprps = DIV_ROUND_UP(length, page_size);

	nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);

	if (nprps <= (256 / 8)) {

		pool = dev->prp_small_pool;

		iod->npages = 0;
@@ -628,7 +627,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, 
	iod->first_dma = prp_dma;

	i = 0;

	for (;;) {

		if (i == page_size >> 3) {

		if (i == NVME_CTRL_PAGE_SIZE >> 3) {

			__le64 *old_prp_list = prp_list;

			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);

			if (!prp_list)
@@ -639,9 +638,9 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, 
			i = 1;

		}

		prp_list[i++] = cpu_to_le64(dma_addr);

		dma_len -= page_size;

		dma_addr += page_size;

		length -= page_size;

		dma_len -= NVME_CTRL_PAGE_SIZE;

		dma_addr += NVME_CTRL_PAGE_SIZE;

		length -= NVME_CTRL_PAGE_SIZE;

		if (length <= 0)

			break;

		if (dma_len > 0)
@@ -751,8 +750,8 @@ static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev, 
		struct bio_vec *bv)

{

	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);

	unsigned int offset = bv->bv_offset & (dev->ctrl.page_size - 1);

	unsigned int first_prp_len = dev->ctrl.page_size - offset;

	unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1);

	unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset;



	iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);

	if (dma_mapping_error(dev->dev, iod->first_dma))
@@ -794,7 +793,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, 
		struct bio_vec bv = req_bvec(req);



		if (!is_pci_p2pdma_page(bv.bv_page)) {

			if (bv.bv_offset + bv.bv_len <= dev->ctrl.page_size * 2)

			if (bv.bv_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2)

				return nvme_setup_prp_simple(dev, req,

							     &cmnd->rw, &bv);
@@ -1396,12 +1395,12 @@ static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, 
{

	int q_depth = dev->q_depth;

	unsigned q_size_aligned = roundup(q_depth * entry_size,

					  dev->ctrl.page_size);

					  NVME_CTRL_PAGE_SIZE);



	if (q_size_aligned * nr_io_queues > dev->cmb_size) {

		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);



		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);

		mem_per_q = round_down(mem_per_q, NVME_CTRL_PAGE_SIZE);

		q_depth = div_u64(mem_per_q, entry_size);



		/*
@@ -1816,6 +1815,7 @@ static inline void nvme_release_cmb(struct nvme_dev *dev) 


static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)

{

	u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT;

	u64 dma_addr = dev->host_mem_descs_dma;

	struct nvme_command c;

	int ret;
@@ -1824,8 +1824,7 @@ static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits) 
	c.features.opcode	= nvme_admin_set_features;

	c.features.fid		= cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);

	c.features.dword11	= cpu_to_le32(bits);

	c.features.dword12	= cpu_to_le32(dev->host_mem_size >>

					      ilog2(dev->ctrl.page_size));

	c.features.dword12	= cpu_to_le32(host_mem_size);

	c.features.dword13	= cpu_to_le32(lower_32_bits(dma_addr));

	c.features.dword14	= cpu_to_le32(upper_32_bits(dma_addr));

	c.features.dword15	= cpu_to_le32(dev->nr_host_mem_descs);
@@ -1845,7 +1844,7 @@ static void nvme_free_host_mem(struct nvme_dev *dev) 


	for (i = 0; i < dev->nr_host_mem_descs; i++) {

		struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];

		size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size;

		size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE;



		dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i],

			       le64_to_cpu(desc->addr),
@@ -1897,7 +1896,7 @@ static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred, 
			break;



		descs[i].addr = cpu_to_le64(dma_addr);

		descs[i].size = cpu_to_le32(len / dev->ctrl.page_size);

		descs[i].size = cpu_to_le32(len / NVME_CTRL_PAGE_SIZE);

		i++;

	}
@@ -1913,7 +1912,7 @@ static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred, 


out_free_bufs:

	while (--i >= 0) {

		size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size;

		size_t size = le32_to_cpu(descs[i].size) * NVME_CTRL_PAGE_SIZE;



		dma_free_attrs(dev->dev, size, bufs[i],

			       le64_to_cpu(descs[i].addr),

CRA Git | Maintained and supported by SUSTech CRA and CCSE