Commit 21e98932 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.infradead.org/users/willy/linux-nvme 

Pull NVMe driver fixes from Matthew Wilcox:
 "Now that actual hardware has been released (don't have any yet
  myself), people are starting to want some of these fixes merged."

Willy doesn't have hardware? Guys...

* git://git.infradead.org/users/willy/linux-nvme:
  NVMe: Cancel outstanding IOs on queue deletion
  NVMe: Free admin queue memory on initialisation failure
  NVMe: Use ida for nvme device instance
  NVMe: Fix whitespace damage in nvme_init
  NVMe: handle allocation failure in nvme_map_user_pages()
  NVMe: Fix uninitialized iod compiler warning
  NVMe: Do not set IO queue depth beyond device max
  NVMe: Set block queue max sectors
  NVMe: use namespace id for nvme_get_features
  NVMe: replace nvme_ns with nvme_dev for user admin
  NVMe: Fix nvme module init when nvme_major is set
  NVMe: Set request queue logical block size
parents 9c603e53 a09115b2

drivers/block/nvme.c

+103 −50
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@ struct nvme_dev { 
	char serial[20];

	char model[40];

	char firmware_rev[8];

	u32 max_hw_sectors;

};



/*
@@ -835,15 +836,15 @@ static int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns, 
}



static int nvme_get_features(struct nvme_dev *dev, unsigned fid,

				unsigned dword11, dma_addr_t dma_addr)

				unsigned nsid, dma_addr_t dma_addr)

{

	struct nvme_command c;



	memset(&c, 0, sizeof(c));

	c.features.opcode = nvme_admin_get_features;

	c.features.nsid = cpu_to_le32(nsid);

	c.features.prp1 = cpu_to_le64(dma_addr);

	c.features.fid = cpu_to_le32(fid);

	c.features.dword11 = cpu_to_le32(dword11);



	return nvme_submit_admin_cmd(dev, &c, NULL);

}
@@ -862,11 +863,51 @@ static int nvme_set_features(struct nvme_dev *dev, unsigned fid, 
	return nvme_submit_admin_cmd(dev, &c, result);

}



/**

 * nvme_cancel_ios - Cancel outstanding I/Os

 * @queue: The queue to cancel I/Os on

 * @timeout: True to only cancel I/Os which have timed out

 */

static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)

{

	int depth = nvmeq->q_depth - 1;

	struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);

	unsigned long now = jiffies;

	int cmdid;



	for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {

		void *ctx;

		nvme_completion_fn fn;

		static struct nvme_completion cqe = {

			.status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1,

		};



		if (timeout && !time_after(now, info[cmdid].timeout))

			continue;

		dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);

		ctx = cancel_cmdid(nvmeq, cmdid, &fn);

		fn(nvmeq->dev, ctx, &cqe);

	}

}



static void nvme_free_queue_mem(struct nvme_queue *nvmeq)

{

	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),

				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);

	dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),

					nvmeq->sq_cmds, nvmeq->sq_dma_addr);

	kfree(nvmeq);

}



static void nvme_free_queue(struct nvme_dev *dev, int qid)

{

	struct nvme_queue *nvmeq = dev->queues[qid];

	int vector = dev->entry[nvmeq->cq_vector].vector;



	spin_lock_irq(&nvmeq->q_lock);

	nvme_cancel_ios(nvmeq, false);

	spin_unlock_irq(&nvmeq->q_lock);



	irq_set_affinity_hint(vector, NULL);

	free_irq(vector, nvmeq);
@@ -876,18 +917,15 @@ static void nvme_free_queue(struct nvme_dev *dev, int qid) 
		adapter_delete_cq(dev, qid);

	}



	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),

				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);

	dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),

					nvmeq->sq_cmds, nvmeq->sq_dma_addr);

	kfree(nvmeq);

	nvme_free_queue_mem(nvmeq);

}



static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,

							int depth, int vector)

{

	struct device *dmadev = &dev->pci_dev->dev;

	unsigned extra = (depth / 8) + (depth * sizeof(struct nvme_cmd_info));

	unsigned extra = DIV_ROUND_UP(depth, 8) + (depth *

						sizeof(struct nvme_cmd_info));

	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);

	if (!nvmeq)

		return NULL;
@@ -975,7 +1013,7 @@ static __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, 


static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev)

{

	int result;

	int result = 0;

	u32 aqa;

	u64 cap;

	unsigned long timeout;
@@ -1005,15 +1043,20 @@ static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev) 
	timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;

	dev->db_stride = NVME_CAP_STRIDE(cap);



	while (!(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {

	while (!result && !(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {

		msleep(100);

		if (fatal_signal_pending(current))

			return -EINTR;

			result = -EINTR;

		if (time_after(jiffies, timeout)) {

			dev_err(&dev->pci_dev->dev,

				"Device not ready; aborting initialisation\n");

			return -ENODEV;

			result = -ENODEV;

		}

	}



	if (result) {

		nvme_free_queue_mem(nvmeq);

		return result;

	}



	result = queue_request_irq(dev, nvmeq, "nvme admin");
@@ -1037,6 +1080,8 @@ static struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write, 
	offset = offset_in_page(addr);

	count = DIV_ROUND_UP(offset + length, PAGE_SIZE);

	pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);

	if (!pages)

		return ERR_PTR(-ENOMEM);



	err = get_user_pages_fast(addr, count, 1, pages);

	if (err < count) {
@@ -1146,14 +1191,13 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 
	return status;

}



static int nvme_user_admin_cmd(struct nvme_ns *ns,

static int nvme_user_admin_cmd(struct nvme_dev *dev,

					struct nvme_admin_cmd __user *ucmd)

{

	struct nvme_dev *dev = ns->dev;

	struct nvme_admin_cmd cmd;

	struct nvme_command c;

	int status, length;

	struct nvme_iod *iod;

	struct nvme_iod *uninitialized_var(iod);



	if (!capable(CAP_SYS_ADMIN))

		return -EACCES;
@@ -1204,7 +1248,7 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 
	case NVME_IOCTL_ID:

		return ns->ns_id;

	case NVME_IOCTL_ADMIN_CMD:

		return nvme_user_admin_cmd(ns, (void __user *)arg);

		return nvme_user_admin_cmd(ns->dev, (void __user *)arg);

	case NVME_IOCTL_SUBMIT_IO:

		return nvme_submit_io(ns, (void __user *)arg);

	default:
@@ -1218,26 +1262,6 @@ static const struct block_device_operations nvme_fops = { 
	.compat_ioctl	= nvme_ioctl,

};



static void nvme_timeout_ios(struct nvme_queue *nvmeq)

{

	int depth = nvmeq->q_depth - 1;

	struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);

	unsigned long now = jiffies;

	int cmdid;



	for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {

		void *ctx;

		nvme_completion_fn fn;

		static struct nvme_completion cqe = { .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1, };



		if (!time_after(now, info[cmdid].timeout))

			continue;

		dev_warn(nvmeq->q_dmadev, "Timing out I/O %d\n", cmdid);

		ctx = cancel_cmdid(nvmeq, cmdid, &fn);

		fn(nvmeq->dev, ctx, &cqe);

	}

}



static void nvme_resubmit_bios(struct nvme_queue *nvmeq)

{

	while (bio_list_peek(&nvmeq->sq_cong)) {
@@ -1269,7 +1293,7 @@ static int nvme_kthread(void *data) 
				spin_lock_irq(&nvmeq->q_lock);

				if (nvme_process_cq(nvmeq))

					printk("process_cq did something\n");

				nvme_timeout_ios(nvmeq);

				nvme_cancel_ios(nvmeq, true);

				nvme_resubmit_bios(nvmeq);

				spin_unlock_irq(&nvmeq->q_lock);

			}
@@ -1339,6 +1363,9 @@ static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid, 
	ns->disk = disk;

	lbaf = id->flbas & 0xf;

	ns->lba_shift = id->lbaf[lbaf].ds;

	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);

	if (dev->max_hw_sectors)

		blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);



	disk->major = nvme_major;

	disk->minors = NVME_MINORS;
@@ -1383,7 +1410,7 @@ static int set_queue_count(struct nvme_dev *dev, int count) 


static int __devinit nvme_setup_io_queues(struct nvme_dev *dev)

{

	int result, cpu, i, nr_io_queues, db_bar_size;

	int result, cpu, i, nr_io_queues, db_bar_size, q_depth;



	nr_io_queues = num_online_cpus();

	result = set_queue_count(dev, nr_io_queues);
@@ -1429,9 +1456,10 @@ static int __devinit nvme_setup_io_queues(struct nvme_dev *dev) 
		cpu = cpumask_next(cpu, cpu_online_mask);

	}



	q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,

								NVME_Q_DEPTH);

	for (i = 0; i < nr_io_queues; i++) {

		dev->queues[i + 1] = nvme_create_queue(dev, i + 1,

							NVME_Q_DEPTH, i);

		dev->queues[i + 1] = nvme_create_queue(dev, i + 1, q_depth, i);

		if (IS_ERR(dev->queues[i + 1]))

			return PTR_ERR(dev->queues[i + 1]);

		dev->queue_count++;
@@ -1480,6 +1508,10 @@ static int __devinit nvme_dev_add(struct nvme_dev *dev) 
	memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));

	memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));

	memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));

	if (ctrl->mdts) {

		int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;

		dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);

	}



	id_ns = mem;

	for (i = 1; i <= nn; i++) {
@@ -1523,8 +1555,6 @@ static int nvme_dev_remove(struct nvme_dev *dev) 
	list_del(&dev->node);

	spin_unlock(&dev_list_lock);



	/* TODO: wait all I/O finished or cancel them */



	list_for_each_entry_safe(ns, next, &dev->namespaces, list) {

		list_del(&ns->list);

		del_gendisk(ns->disk);
@@ -1560,15 +1590,33 @@ static void nvme_release_prp_pools(struct nvme_dev *dev) 
	dma_pool_destroy(dev->prp_small_pool);

}



/* XXX: Use an ida or something to let remove / add work correctly */

static void nvme_set_instance(struct nvme_dev *dev)

static DEFINE_IDA(nvme_instance_ida);



static int nvme_set_instance(struct nvme_dev *dev)

{

	static int instance;

	dev->instance = instance++;

	int instance, error;



	do {

		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))

			return -ENODEV;



		spin_lock(&dev_list_lock);

		error = ida_get_new(&nvme_instance_ida, &instance);

		spin_unlock(&dev_list_lock);

	} while (error == -EAGAIN);



	if (error)

		return -ENODEV;



	dev->instance = instance;

	return 0;

}



static void nvme_release_instance(struct nvme_dev *dev)

{

	spin_lock(&dev_list_lock);

	ida_remove(&nvme_instance_ida, dev->instance);

	spin_unlock(&dev_list_lock);

}



static int __devinit nvme_probe(struct pci_dev *pdev,
@@ -1601,7 +1649,10 @@ static int __devinit nvme_probe(struct pci_dev *pdev, 
	pci_set_drvdata(pdev, dev);

	dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));

	dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));

	nvme_set_instance(dev);

	result = nvme_set_instance(dev);

	if (result)

		goto disable;



	dev->entry[0].vector = pdev->irq;



	result = nvme_setup_prp_pools(dev);
@@ -1704,15 +1755,17 @@ static struct pci_driver nvme_driver = { 


static int __init nvme_init(void)

{

	int result = -EBUSY;

	int result;



	nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");

	if (IS_ERR(nvme_thread))

		return PTR_ERR(nvme_thread);



	nvme_major = register_blkdev(nvme_major, "nvme");

	if (nvme_major <= 0)

	result = register_blkdev(nvme_major, "nvme");

	if (result < 0)

		goto kill_kthread;

	else if (result > 0)

		nvme_major = result;



	result = pci_register_driver(&nvme_driver);

	if (result)

include/linux/nvme.h

+2 −0
Original line number Diff line number Diff line
@@ -35,8 +35,10 @@ struct nvme_bar { 
	__u64			acq;	/* Admin CQ Base Address */

};



#define NVME_CAP_MQES(cap)	((cap) & 0xffff)

#define NVME_CAP_TIMEOUT(cap)	(((cap) >> 24) & 0xff)

#define NVME_CAP_STRIDE(cap)	(((cap) >> 32) & 0xf)

#define NVME_CAP_MPSMIN(cap)	(((cap) >> 48) & 0xf)



enum {

	NVME_CC_ENABLE		= 1 << 0,

CRA Git | Maintained and supported by SUSTech CRA and CCSE