Commit 73cc4bb0 authored by David Vrabel's avatar David Vrabel Committed by Konrad Rzeszutek Wilk
Browse files

xen/evtchn: improve scalability by using per-user locks 



The global array of port users and the port_user_lock limits
scalability of the evtchn device.  Instead of the global array lookup,
use a per-use (per-fd) tree of event channels bound by that user and
protect the tree with a per-user lock.

This is also a prerequiste for extended the number of supported event
channels, by removing the fixed size, per-event channel array.

Signed-off-by: default avatarDavid Vrabel <david.vrabel@citrix.com>
Signed-off-by: default avatarKonrad Rzeszutek Wilk <konrad.wilk@oracle.com>
parent 65a45fa2

drivers/xen/evtchn.c

+112 −80
Original line number Diff line number Diff line
@@ -57,6 +57,7 @@ 


struct per_user_data {

	struct mutex bind_mutex; /* serialize bind/unbind operations */

	struct rb_root evtchns;



	/* Notification ring, accessed via /dev/xen/evtchn. */

#define EVTCHN_RING_SIZE     (PAGE_SIZE / sizeof(evtchn_port_t))
@@ -64,6 +65,7 @@ struct per_user_data { 
	evtchn_port_t *ring;

	unsigned int ring_cons, ring_prod, ring_overflow;

	struct mutex ring_cons_mutex; /* protect against concurrent readers */

	spinlock_t ring_prod_lock; /* product against concurrent interrupts */



	/* Processes wait on this queue when ring is empty. */

	wait_queue_head_t evtchn_wait;
@@ -71,54 +73,79 @@ struct per_user_data { 
	const char *name;

};



/*

 * Who's bound to each port?  This is logically an array of struct

 * per_user_data *, but we encode the current enabled-state in bit 0.

 */

static unsigned long *port_user;

static DEFINE_SPINLOCK(port_user_lock); /* protects port_user[] and ring_prod */

struct user_evtchn {

	struct rb_node node;

	struct per_user_data *user;

	unsigned port;

	bool enabled;

};



static inline struct per_user_data *get_port_user(unsigned port)

static int add_evtchn(struct per_user_data *u, struct user_evtchn *evtchn)

{

	return (struct per_user_data *)(port_user[port] & ~1);

	struct rb_node **new = &(u->evtchns.rb_node), *parent = NULL;



	while (*new) {

		struct user_evtchn *this;



		this = container_of(*new, struct user_evtchn, node);



		parent = *new;

		if (this->port < evtchn->port)

			new = &((*new)->rb_left);

		else if (this->port > evtchn->port)

			new = &((*new)->rb_right);

		else

			return -EEXIST;

	}



static inline void set_port_user(unsigned port, struct per_user_data *u)

{

	port_user[port] = (unsigned long)u;

	/* Add new node and rebalance tree. */

	rb_link_node(&evtchn->node, parent, new);

	rb_insert_color(&evtchn->node, &u->evtchns);



	return 0;

}



static inline bool get_port_enabled(unsigned port)

static void del_evtchn(struct per_user_data *u, struct user_evtchn *evtchn)

{

	return port_user[port] & 1;

	rb_erase(&evtchn->node, &u->evtchns);

	kfree(evtchn);

}



static inline void set_port_enabled(unsigned port, bool enabled)

static struct user_evtchn *find_evtchn(struct per_user_data *u, unsigned port)

{

	if (enabled)

		port_user[port] |= 1;

	struct rb_node *node = u->evtchns.rb_node;



	while (node) {

		struct user_evtchn *evtchn;



		evtchn = container_of(node, struct user_evtchn, node);



		if (evtchn->port < port)

			node = node->rb_left;

		else if (evtchn->port > port)

			node = node->rb_right;

		else

		port_user[port] &= ~1;

			return evtchn;

	}

	return NULL;

}



static irqreturn_t evtchn_interrupt(int irq, void *data)

{

	unsigned int port = (unsigned long)data;

	struct per_user_data *u;

	struct user_evtchn *evtchn = data;

	struct per_user_data *u = evtchn->user;



	spin_lock(&port_user_lock);



	u = get_port_user(port);



	WARN(!get_port_enabled(port),

	WARN(!evtchn->enabled,

	     "Interrupt for port %d, but apparently not enabled; per-user %p\n",

	     port, u);

	     evtchn->port, u);



	disable_irq_nosync(irq);

	set_port_enabled(port, false);

	evtchn->enabled = false;



	spin_lock(&u->ring_prod_lock);



	if ((u->ring_prod - u->ring_cons) < EVTCHN_RING_SIZE) {

		u->ring[EVTCHN_RING_MASK(u->ring_prod)] = port;

		u->ring[EVTCHN_RING_MASK(u->ring_prod)] = evtchn->port;

		wmb(); /* Ensure ring contents visible */

		if (u->ring_cons == u->ring_prod++) {

			wake_up_interruptible(&u->evtchn_wait);
@@ -128,7 +155,7 @@ static irqreturn_t evtchn_interrupt(int irq, void *data) 
	} else

		u->ring_overflow = 1;



	spin_unlock(&port_user_lock);

	spin_unlock(&u->ring_prod_lock);



	return IRQ_HANDLED;

}
@@ -229,20 +256,20 @@ static ssize_t evtchn_write(struct file *file, const char __user *buf, 
	if (copy_from_user(kbuf, buf, count) != 0)

		goto out;



	spin_lock_irq(&port_user_lock);

	mutex_lock(&u->bind_mutex);



	for (i = 0; i < (count/sizeof(evtchn_port_t)); i++) {

		unsigned port = kbuf[i];

		struct user_evtchn *evtchn;



		if (port < NR_EVENT_CHANNELS &&

		    get_port_user(port) == u &&

		    !get_port_enabled(port)) {

			set_port_enabled(port, true);

		evtchn = find_evtchn(u, port);

		if (evtchn && !evtchn->enabled) {

			evtchn->enabled = true;

			enable_irq(irq_from_evtchn(port));

		}

	}



	spin_unlock_irq(&port_user_lock);

	mutex_unlock(&u->bind_mutex);



	rc = count;
@@ -253,6 +280,8 @@ static ssize_t evtchn_write(struct file *file, const char __user *buf, 


static int evtchn_bind_to_user(struct per_user_data *u, int port)

{

	struct user_evtchn *evtchn;

	struct evtchn_close close;

	int rc = 0;



	/*
@@ -263,35 +292,47 @@ static int evtchn_bind_to_user(struct per_user_data *u, int port) 
	 * interrupt handler yet, and our caller has already

	 * serialized bind operations.)

	 */

	BUG_ON(get_port_user(port) != NULL);

	set_port_user(port, u);

	set_port_enabled(port, true); /* start enabled */



	evtchn = kzalloc(sizeof(*evtchn), GFP_KERNEL);

	if (!evtchn)

		return -ENOMEM;



	evtchn->user = u;

	evtchn->port = port;

	evtchn->enabled = true; /* start enabled */



	rc = add_evtchn(u, evtchn);

	if (rc < 0)

		goto err;



	rc = bind_evtchn_to_irqhandler(port, evtchn_interrupt, IRQF_DISABLED,

				       u->name, (void *)(unsigned long)port);

	if (rc >= 0)

				       u->name, evtchn);

	if (rc < 0)

		goto err;



	rc = evtchn_make_refcounted(port);

	else {

	return rc;



err:

	/* bind failed, should close the port now */

		struct evtchn_close close;

	close.port = port;

	if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)

		BUG();

		set_port_user(port, NULL);

	}



	del_evtchn(u, evtchn);

	kfree(evtchn);

	return rc;

}



static void evtchn_unbind_from_user(struct per_user_data *u, int port)

static void evtchn_unbind_from_user(struct per_user_data *u,

				    struct user_evtchn *evtchn)

{

	int irq = irq_from_evtchn(port);

	int irq = irq_from_evtchn(evtchn->port);



	BUG_ON(irq < 0);



	unbind_from_irqhandler(irq, (void *)(unsigned long)port);

	unbind_from_irqhandler(irq, evtchn);



	set_port_user(port, NULL);

	del_evtchn(u, evtchn);

}



static long evtchn_ioctl(struct file *file,
@@ -370,6 +411,7 @@ static long evtchn_ioctl(struct file *file, 


	case IOCTL_EVTCHN_UNBIND: {

		struct ioctl_evtchn_unbind unbind;

		struct user_evtchn *evtchn;



		rc = -EFAULT;

		if (copy_from_user(&unbind, uarg, sizeof(unbind)))
@@ -380,29 +422,27 @@ static long evtchn_ioctl(struct file *file, 
			break;



		rc = -ENOTCONN;

		if (get_port_user(unbind.port) != u)

		evtchn = find_evtchn(u, unbind.port);

		if (!evtchn)

			break;



		disable_irq(irq_from_evtchn(unbind.port));



		evtchn_unbind_from_user(u, unbind.port);



		evtchn_unbind_from_user(u, evtchn);

		rc = 0;

		break;

	}



	case IOCTL_EVTCHN_NOTIFY: {

		struct ioctl_evtchn_notify notify;

		struct user_evtchn *evtchn;



		rc = -EFAULT;

		if (copy_from_user(&notify, uarg, sizeof(notify)))

			break;



		if (notify.port >= NR_EVENT_CHANNELS) {

			rc = -EINVAL;

		} else if (get_port_user(notify.port) != u) {

		rc = -ENOTCONN;

		} else {

		evtchn = find_evtchn(u, notify.port);

		if (evtchn) {

			notify_remote_via_evtchn(notify.port);

			rc = 0;

		}
@@ -412,9 +452,9 @@ static long evtchn_ioctl(struct file *file, 
	case IOCTL_EVTCHN_RESET: {

		/* Initialise the ring to empty. Clear errors. */

		mutex_lock(&u->ring_cons_mutex);

		spin_lock_irq(&port_user_lock);

		spin_lock_irq(&u->ring_prod_lock);

		u->ring_cons = u->ring_prod = u->ring_overflow = 0;

		spin_unlock_irq(&port_user_lock);

		spin_unlock_irq(&u->ring_prod_lock);

		mutex_unlock(&u->ring_cons_mutex);

		rc = 0;

		break;
@@ -473,6 +513,7 @@ static int evtchn_open(struct inode *inode, struct file *filp) 


	mutex_init(&u->bind_mutex);

	mutex_init(&u->ring_cons_mutex);

	spin_lock_init(&u->ring_prod_lock);



	filp->private_data = u;
@@ -481,15 +522,15 @@ static int evtchn_open(struct inode *inode, struct file *filp) 


static int evtchn_release(struct inode *inode, struct file *filp)

{

	int i;

	struct per_user_data *u = filp->private_data;

	struct rb_node *node;



	for (i = 0; i < NR_EVENT_CHANNELS; i++) {

		if (get_port_user(i) != u)

			continue;

	while ((node = u->evtchns.rb_node)) {

		struct user_evtchn *evtchn;



		disable_irq(irq_from_evtchn(i));

		evtchn_unbind_from_user(get_port_user(i), i);

		evtchn = rb_entry(node, struct user_evtchn, node);

		disable_irq(irq_from_evtchn(evtchn->port));

		evtchn_unbind_from_user(u, evtchn);

	}



	free_page((unsigned long)u->ring);
@@ -523,12 +564,6 @@ static int __init evtchn_init(void) 
	if (!xen_domain())

		return -ENODEV;



	port_user = kcalloc(NR_EVENT_CHANNELS, sizeof(*port_user), GFP_KERNEL);

	if (port_user == NULL)

		return -ENOMEM;



	spin_lock_init(&port_user_lock);



	/* Create '/dev/xen/evtchn'. */

	err = misc_register(&evtchn_miscdev);

	if (err != 0) {
@@ -543,9 +578,6 @@ static int __init evtchn_init(void) 


static void __exit evtchn_cleanup(void)

{

	kfree(port_user);

	port_user = NULL;



	misc_deregister(&evtchn_miscdev);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE