Commit 7838f994 authored by Robin Holt's avatar Robin Holt Committed by Linus Torvalds
Browse files

drivers/misc/sgi-xp/xpc_uv.c: SGI XPC fails to load when cpu 0 is out of IRQ resources 



On many of our larger systems, CPU 0 has had all of its IRQ resources
consumed before XPC loads.  Worst cases on machines with multiple 10
GigE cards and multiple IB cards have depleted the entire first socket
of IRQs.

This patch makes selecting the node upon which IRQs are allocated (as
well as all the other GRU Message Queue structures) specifiable as a
module load param and has a default behavior of searching all nodes/cpus
for an available resources.

[akpm@linux-foundation.org: fix build: include cpu.h and module.h]
Signed-off-by: default avatarRobin Holt <holt@sgi.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c3a5ce04

drivers/misc/sgi-xp/xpc_uv.c

+65 −19
Original line number Diff line number Diff line
@@ -18,6 +18,8 @@ 
#include <linux/interrupt.h>

#include <linux/delay.h>

#include <linux/device.h>

#include <linux/cpu.h>

#include <linux/module.h>

#include <linux/err.h>

#include <linux/slab.h>

#include <asm/uv/uv_hub.h>
@@ -59,6 +61,8 @@ static struct xpc_heartbeat_uv *xpc_heartbeat_uv; 
					 XPC_NOTIFY_MSG_SIZE_UV)

#define XPC_NOTIFY_IRQ_NAME		"xpc_notify"



static int xpc_mq_node = -1;



static struct xpc_gru_mq_uv *xpc_activate_mq_uv;

static struct xpc_gru_mq_uv *xpc_notify_mq_uv;
@@ -109,11 +113,8 @@ xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name) 
#if defined CONFIG_X86_64

	mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset,

			UV_AFFINITY_CPU);

	if (mq->irq < 0) {

		dev_err(xpc_part, "uv_setup_irq() returned error=%d\n",

			-mq->irq);

	if (mq->irq < 0)

		return mq->irq;

	}



	mq->mmr_value = uv_read_global_mmr64(mmr_pnode, mq->mmr_offset);
@@ -238,7 +239,8 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name, 
	mq->mmr_blade = uv_cpu_to_blade_id(cpu);



	nid = cpu_to_node(cpu);

	page = alloc_pages_exact_node(nid, GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,

	page = alloc_pages_exact_node(nid,

				      GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,

				      pg_order);

	if (page == NULL) {

		dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d "
@@ -1731,37 +1733,81 @@ static struct xpc_arch_operations xpc_arch_ops_uv = { 
	.notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_uv,

};



int

xpc_init_uv(void)

static int

xpc_init_mq_node(int nid)

{

	xpc_arch_ops = xpc_arch_ops_uv;

	int cpu;



	if (sizeof(struct xpc_notify_mq_msghdr_uv) > XPC_MSG_HDR_MAX_SIZE) {

		dev_err(xpc_part, "xpc_notify_mq_msghdr_uv is larger than %d\n",

			XPC_MSG_HDR_MAX_SIZE);

		return -E2BIG;

	}

	get_online_cpus();



	xpc_activate_mq_uv = xpc_create_gru_mq_uv(XPC_ACTIVATE_MQ_SIZE_UV, 0,

	for_each_cpu(cpu, cpumask_of_node(nid)) {

		xpc_activate_mq_uv =

			xpc_create_gru_mq_uv(XPC_ACTIVATE_MQ_SIZE_UV, nid,

					     XPC_ACTIVATE_IRQ_NAME,

					     xpc_handle_activate_IRQ_uv);

	if (IS_ERR(xpc_activate_mq_uv))

		if (!IS_ERR(xpc_activate_mq_uv))

			break;

	}

	if (IS_ERR(xpc_activate_mq_uv)) {

		put_online_cpus();

		return PTR_ERR(xpc_activate_mq_uv);

	}



	xpc_notify_mq_uv = xpc_create_gru_mq_uv(XPC_NOTIFY_MQ_SIZE_UV, 0,

	for_each_cpu(cpu, cpumask_of_node(nid)) {

		xpc_notify_mq_uv =

			xpc_create_gru_mq_uv(XPC_NOTIFY_MQ_SIZE_UV, nid,

					     XPC_NOTIFY_IRQ_NAME,

					     xpc_handle_notify_IRQ_uv);

		if (!IS_ERR(xpc_notify_mq_uv))

			break;

	}

	if (IS_ERR(xpc_notify_mq_uv)) {

		xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);

		put_online_cpus();

		return PTR_ERR(xpc_notify_mq_uv);

	}



	put_online_cpus();

	return 0;

}



int

xpc_init_uv(void)

{

	int nid;

	int ret = 0;



	xpc_arch_ops = xpc_arch_ops_uv;



	if (sizeof(struct xpc_notify_mq_msghdr_uv) > XPC_MSG_HDR_MAX_SIZE) {

		dev_err(xpc_part, "xpc_notify_mq_msghdr_uv is larger than %d\n",

			XPC_MSG_HDR_MAX_SIZE);

		return -E2BIG;

	}



	if (xpc_mq_node < 0)

		for_each_online_node(nid) {

			ret = xpc_init_mq_node(nid);



			if (!ret)

				break;

		}

	else

		ret = xpc_init_mq_node(xpc_mq_node);



	if (ret < 0)

		dev_err(xpc_part, "xpc_init_mq_node() returned error=%d\n",

			-ret);



	return ret;

}



void

xpc_exit_uv(void)

{

	xpc_destroy_gru_mq_uv(xpc_notify_mq_uv);

	xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);

}



module_param(xpc_mq_node, int, 0);

MODULE_PARM_DESC(xpc_mq_node, "Node number on which to allocate message queues.");

CRA Git | Maintained and supported by SUSTech CRA and CCSE