Commit 2a2d1382 authored by Andy Lutomirski's avatar Andy Lutomirski Committed by Michael S. Tsirkin
Browse files

virtio: Add improved queue allocation API 



This leaves vring_new_virtqueue alone for compatbility, but it
adds two new improved APIs:

vring_create_virtqueue: Creates a virtqueue backed by automatically
allocated coherent memory.  (Some day it this could be extended to
support non-coherent memory, too, if there ends up being a platform
on which it's worthwhile.)

__vring_new_virtqueue: Creates a virtqueue with a manually-specified
layout.  This should allow mic_virtio to work much more cleanly.

Signed-off-by: default avatarAndy Lutomirski <luto@kernel.org>
Signed-off-by: default avatarMichael S. Tsirkin <mst@redhat.com>
parent 780bc790

drivers/virtio/virtio_ring.c

+179 −30
Original line number Diff line number Diff line
@@ -95,6 +95,11 @@ struct vring_virtqueue { 
	/* How to notify other side. FIXME: commonalize hcalls! */

	bool (*notify)(struct virtqueue *vq);



	/* DMA, allocation, and size information */

	bool we_own_ring;

	size_t queue_size_in_bytes;

	dma_addr_t queue_dma_addr;



#ifdef DEBUG

	/* They're supposed to lock for us. */

	unsigned int in_use;
@@ -878,36 +883,31 @@ irqreturn_t vring_interrupt(int irq, void *_vq) 
}

EXPORT_SYMBOL_GPL(vring_interrupt);



struct virtqueue *vring_new_virtqueue(unsigned int index,

				      unsigned int num,

				      unsigned int vring_align,

struct virtqueue *__vring_new_virtqueue(unsigned int index,

					struct vring vring,

					struct virtio_device *vdev,

					bool weak_barriers,

				      void *pages,

					bool (*notify)(struct virtqueue *),

					void (*callback)(struct virtqueue *),

					const char *name)

{

	struct vring_virtqueue *vq;

	unsigned int i;

	struct vring_virtqueue *vq;



	/* We assume num is a power of 2. */

	if (num & (num - 1)) {

		dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);

		return NULL;

	}



	vq = kmalloc(sizeof(*vq) + num * sizeof(struct vring_desc_state),

	vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state),

		     GFP_KERNEL);

	if (!vq)

		return NULL;



	vring_init(&vq->vring, num, pages, vring_align);

	vq->vring = vring;

	vq->vq.callback = callback;

	vq->vq.vdev = vdev;

	vq->vq.name = name;

	vq->vq.num_free = num;

	vq->vq.num_free = vring.num;

	vq->vq.index = index;

	vq->we_own_ring = false;

	vq->queue_dma_addr = 0;

	vq->queue_size_in_bytes = 0;

	vq->notify = notify;

	vq->weak_barriers = weak_barriers;

	vq->broken = false;
@@ -932,18 +932,145 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, 


	/* Put everything in free lists. */

	vq->free_head = 0;

	for (i = 0; i < num-1; i++)

	for (i = 0; i < vring.num-1; i++)

		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);

	memset(vq->desc_state, 0, num * sizeof(struct vring_desc_state));

	memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state));



	return &vq->vq;

}

EXPORT_SYMBOL_GPL(__vring_new_virtqueue);



static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,

			      dma_addr_t *dma_handle, gfp_t flag)

{

	if (vring_use_dma_api(vdev)) {

		return dma_alloc_coherent(vdev->dev.parent, size,

					  dma_handle, flag);

	} else {

		void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);

		if (queue) {

			phys_addr_t phys_addr = virt_to_phys(queue);

			*dma_handle = (dma_addr_t)phys_addr;



			/*

			 * Sanity check: make sure we dind't truncate

			 * the address.  The only arches I can find that

			 * have 64-bit phys_addr_t but 32-bit dma_addr_t

			 * are certain non-highmem MIPS and x86

			 * configurations, but these configurations

			 * should never allocate physical pages above 32

			 * bits, so this is fine.  Just in case, throw a

			 * warning and abort if we end up with an

			 * unrepresentable address.

			 */

			if (WARN_ON_ONCE(*dma_handle != phys_addr)) {

				free_pages_exact(queue, PAGE_ALIGN(size));

				return NULL;

			}

		}

		return queue;

	}

}



static void vring_free_queue(struct virtio_device *vdev, size_t size,

			     void *queue, dma_addr_t dma_handle)

{

	if (vring_use_dma_api(vdev)) {

		dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);

	} else {

		free_pages_exact(queue, PAGE_ALIGN(size));

	}

}



struct virtqueue *vring_create_virtqueue(

	unsigned int index,

	unsigned int num,

	unsigned int vring_align,

	struct virtio_device *vdev,

	bool weak_barriers,

	bool may_reduce_num,

	bool (*notify)(struct virtqueue *),

	void (*callback)(struct virtqueue *),

	const char *name)

{

	struct virtqueue *vq;

	void *queue;

	dma_addr_t dma_addr;

	size_t queue_size_in_bytes;

	struct vring vring;



	/* We assume num is a power of 2. */

	if (num & (num - 1)) {

		dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);

		return NULL;

	}



	/* TODO: allocate each queue chunk individually */

	for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {

		queue = vring_alloc_queue(vdev, vring_size(num, vring_align),

					  &dma_addr,

					  GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);

		if (queue)

			break;

	}



	if (!num)

		return NULL;



	if (!queue) {

		/* Try to get a single page. You are my only hope! */

		queue = vring_alloc_queue(vdev, vring_size(num, vring_align),

					  &dma_addr, GFP_KERNEL|__GFP_ZERO);

	}

	if (!queue)

		return NULL;



	queue_size_in_bytes = vring_size(num, vring_align);

	vring_init(&vring, num, queue, vring_align);



	vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers,

				   notify, callback, name);

	if (!vq) {

		vring_free_queue(vdev, queue_size_in_bytes, queue,

				 dma_addr);

		return NULL;

	}



	to_vvq(vq)->queue_dma_addr = dma_addr;

	to_vvq(vq)->queue_size_in_bytes = queue_size_in_bytes;

	to_vvq(vq)->we_own_ring = true;



	return vq;

}

EXPORT_SYMBOL_GPL(vring_create_virtqueue);



struct virtqueue *vring_new_virtqueue(unsigned int index,

				      unsigned int num,

				      unsigned int vring_align,

				      struct virtio_device *vdev,

				      bool weak_barriers,

				      void *pages,

				      bool (*notify)(struct virtqueue *vq),

				      void (*callback)(struct virtqueue *vq),

				      const char *name)

{

	struct vring vring;

	vring_init(&vring, num, pages, vring_align);

	return __vring_new_virtqueue(index, vring, vdev, weak_barriers,

				     notify, callback, name);

}

EXPORT_SYMBOL_GPL(vring_new_virtqueue);



void vring_del_virtqueue(struct virtqueue *vq)

void vring_del_virtqueue(struct virtqueue *_vq)

{

	list_del(&vq->list);

	kfree(to_vvq(vq));

	struct vring_virtqueue *vq = to_vvq(_vq);



	if (vq->we_own_ring) {

		vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes,

				 vq->vring.desc, vq->queue_dma_addr);

	}

	list_del(&_vq->list);

	kfree(vq);

}

EXPORT_SYMBOL_GPL(vring_del_virtqueue);
@@ -1007,20 +1134,42 @@ void virtio_break_device(struct virtio_device *dev) 
}

EXPORT_SYMBOL_GPL(virtio_break_device);



void *virtqueue_get_avail(struct virtqueue *_vq)

dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)

{

	struct vring_virtqueue *vq = to_vvq(_vq);



	return vq->vring.avail;

	BUG_ON(!vq->we_own_ring);



	return vq->queue_dma_addr;

}

EXPORT_SYMBOL_GPL(virtqueue_get_avail);

EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);



void *virtqueue_get_used(struct virtqueue *_vq)

dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)

{

	struct vring_virtqueue *vq = to_vvq(_vq);



	return vq->vring.used;

	BUG_ON(!vq->we_own_ring);



	return vq->queue_dma_addr +

		((char *)vq->vring.avail - (char *)vq->vring.desc);

}

EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);



dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)

{

	struct vring_virtqueue *vq = to_vvq(_vq);



	BUG_ON(!vq->we_own_ring);



	return vq->queue_dma_addr +

		((char *)vq->vring.used - (char *)vq->vring.desc);

}

EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);



const struct vring *virtqueue_get_vring(struct virtqueue *vq)

{

	return &to_vvq(vq)->vring;

}

EXPORT_SYMBOL_GPL(virtqueue_get_used);

EXPORT_SYMBOL_GPL(virtqueue_get_vring);



MODULE_LICENSE("GPL");

include/linux/virtio.h

+21 −2
Original line number Diff line number Diff line
@@ -75,8 +75,27 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *vq); 


bool virtqueue_is_broken(struct virtqueue *vq);



void *virtqueue_get_avail(struct virtqueue *vq);

void *virtqueue_get_used(struct virtqueue *vq);

const struct vring *virtqueue_get_vring(struct virtqueue *vq);

dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq);

dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq);

dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq);



/*

 * Legacy accessors -- in almost all cases, these are the wrong functions

 * to use.

 */

static inline void *virtqueue_get_desc(struct virtqueue *vq)

{

	return virtqueue_get_vring(vq)->desc;

}

static inline void *virtqueue_get_avail(struct virtqueue *vq)

{

	return virtqueue_get_vring(vq)->avail;

}

static inline void *virtqueue_get_used(struct virtqueue *vq)

{

	return virtqueue_get_vring(vq)->used;

}



/**

 * virtio_device - representation of a device using virtio

include/linux/virtio_ring.h

+35 −0
Original line number Diff line number Diff line
@@ -59,6 +59,35 @@ static inline void virtio_store_mb(bool weak_barriers, 
struct virtio_device;

struct virtqueue;



/*

 * Creates a virtqueue and allocates the descriptor ring.  If

 * may_reduce_num is set, then this may allocate a smaller ring than

 * expected.  The caller should query virtqueue_get_ring_size to learn

 * the actual size of the ring.

 */

struct virtqueue *vring_create_virtqueue(unsigned int index,

					 unsigned int num,

					 unsigned int vring_align,

					 struct virtio_device *vdev,

					 bool weak_barriers,

					 bool may_reduce_num,

					 bool (*notify)(struct virtqueue *vq),

					 void (*callback)(struct virtqueue *vq),

					 const char *name);



/* Creates a virtqueue with a custom layout. */

struct virtqueue *__vring_new_virtqueue(unsigned int index,

					struct vring vring,

					struct virtio_device *vdev,

					bool weak_barriers,

					bool (*notify)(struct virtqueue *),

					void (*callback)(struct virtqueue *),

					const char *name);



/*

 * Creates a virtqueue with a standard layout but a caller-allocated

 * ring.

 */

struct virtqueue *vring_new_virtqueue(unsigned int index,

				      unsigned int num,

				      unsigned int vring_align,
@@ -68,7 +97,13 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, 
				      bool (*notify)(struct virtqueue *vq),

				      void (*callback)(struct virtqueue *vq),

				      const char *name);



/*

 * Destroys a virtqueue.  If created with vring_create_virtqueue, this

 * also frees the ring.

 */

void vring_del_virtqueue(struct virtqueue *vq);



/* Filter out transport-specific feature bits. */

void vring_transport_features(struct virtio_device *vdev);

CRA Git | Maintained and supported by SUSTech CRA and CCSE