Commit a0fedc85 authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge branch 'irq/for-block' of... 

Merge branch 'irq/for-block' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip into for-4.21/block

Pull in the irq affinity commits, that are staged through Thomas's
tree.

* 'irq/for-block' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  genirq/affinity: Add support for allocating interrupt sets
  genirq/affinity: Pass first vector to __irq_build_affinity_masks()
  genirq/affinity: Move two stage affinity spreading into a helper function
  genirq/affinity: Spread IRQs to all available NUMA nodes
parents 9cf2bab6 6da4b3ab

drivers/pci/msi.c

+14 −0
Original line number Diff line number Diff line
@@ -1036,6 +1036,13 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, 
	if (maxvec < minvec)

		return -ERANGE;



	/*

	 * If the caller is passing in sets, we can't support a range of

	 * vectors. The caller needs to handle that.

	 */

	if (affd && affd->nr_sets && minvec != maxvec)

		return -EINVAL;



	if (WARN_ON_ONCE(dev->msi_enabled))

		return -EINVAL;
@@ -1087,6 +1094,13 @@ static int __pci_enable_msix_range(struct pci_dev *dev, 
	if (maxvec < minvec)

		return -ERANGE;



	/*

	 * If the caller is passing in sets, we can't support a range of

	 * supported vectors. The caller needs to handle that.

	 */

	if (affd && affd->nr_sets && minvec != maxvec)

		return -EINVAL;



	if (WARN_ON_ONCE(dev->msix_enabled))

		return -EINVAL;

include/linux/interrupt.h

+4 −0
Original line number Diff line number Diff line
@@ -247,10 +247,14 @@ struct irq_affinity_notify { 
 *			the MSI(-X) vector space

 * @post_vectors:	Don't apply affinity to @post_vectors at end of

 *			the MSI(-X) vector space

 * @nr_sets:		Length of passed in *sets array

 * @sets:		Number of affinitized sets

 */

struct irq_affinity {

	int	pre_vectors;

	int	post_vectors;

	int	nr_sets;

	int	*sets;

};



#if defined(CONFIG_SMP)

kernel/irq/affinity.c

+100 −48
Original line number Diff line number Diff line
@@ -94,15 +94,15 @@ static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask, 
	return nodes;

}



static int irq_build_affinity_masks(const struct irq_affinity *affd,

				    int startvec, int numvecs,

static int __irq_build_affinity_masks(const struct irq_affinity *affd,

				    int startvec, int numvecs, int firstvec,

				    cpumask_var_t *node_to_cpumask,

				    const struct cpumask *cpu_mask,

				    struct cpumask *nmsk,

				    struct cpumask *masks)

{

	int n, nodes, cpus_per_vec, extra_vecs, done = 0;

	int last_affv = affd->pre_vectors + numvecs;

	int last_affv = firstvec + numvecs;

	int curvec = startvec;

	nodemask_t nodemsk = NODE_MASK_NONE;
@@ -117,12 +117,11 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, 
	 */

	if (numvecs <= nodes) {

		for_each_node_mask(n, nodemsk) {

			cpumask_copy(masks + curvec, node_to_cpumask[n]);

			if (++done == numvecs)

				break;

			cpumask_or(masks + curvec, masks + curvec, node_to_cpumask[n]);

			if (++curvec == last_affv)

				curvec = affd->pre_vectors;

				curvec = firstvec;

		}

		done = numvecs;

		goto out;

	}
@@ -130,7 +129,7 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, 
		int ncpus, v, vecs_to_assign, vecs_per_node;



		/* Spread the vectors per node */

		vecs_per_node = (numvecs - (curvec - affd->pre_vectors)) / nodes;

		vecs_per_node = (numvecs - (curvec - firstvec)) / nodes;



		/* Get the cpus on this node which are in the mask */

		cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
@@ -158,7 +157,7 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, 
		if (done >= numvecs)

			break;

		if (curvec >= last_affv)

			curvec = affd->pre_vectors;

			curvec = firstvec;

		--nodes;

	}
@@ -166,6 +165,62 @@ out: 
	return done;

}



/*

 * build affinity in two stages:

 *	1) spread present CPU on these vectors

 *	2) spread other possible CPUs on these vectors

 */

static int irq_build_affinity_masks(const struct irq_affinity *affd,

				    int startvec, int numvecs, int firstvec,

				    cpumask_var_t *node_to_cpumask,

				    struct cpumask *masks)

{

	int curvec = startvec, nr_present, nr_others;

	int ret = -ENOMEM;

	cpumask_var_t nmsk, npresmsk;



	if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))

			return ret;



	if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL))

			goto fail;



	ret = 0;

	/* Stabilize the cpumasks */

	get_online_cpus();

	build_node_to_cpumask(node_to_cpumask);



	/* Spread on present CPUs starting from affd->pre_vectors */

	nr_present = __irq_build_affinity_masks(affd, curvec, numvecs,

						firstvec, node_to_cpumask,

						cpu_present_mask, nmsk, masks);



	/*

	 * Spread on non present CPUs starting from the next vector to be

	 * handled. If the spreading of present CPUs already exhausted the

	 * vector space, assign the non present CPUs to the already spread

	 * out vectors.

	 */

	if (nr_present >= numvecs)

		curvec = firstvec;

	else

		curvec = firstvec + nr_present;

	cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask);

	nr_others = __irq_build_affinity_masks(affd, curvec, numvecs,

					       firstvec, node_to_cpumask,

					       npresmsk, nmsk, masks);

	put_online_cpus();



	if (nr_present < numvecs)

			WARN_ON(nr_present + nr_others < numvecs);



	free_cpumask_var(npresmsk);



 fail:

	free_cpumask_var(nmsk);

	return ret;

}



/**

 * irq_create_affinity_masks - Create affinity masks for multiqueue spreading

 * @nvecs:	The total number of vectors
@@ -178,8 +233,9 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) 
{

	int affvecs = nvecs - affd->pre_vectors - affd->post_vectors;

	int curvec, usedvecs;

	cpumask_var_t nmsk, npresmsk, *node_to_cpumask;

	cpumask_var_t *node_to_cpumask;

	struct cpumask *masks = NULL;

	int i, nr_sets;



	/*

	 * If there aren't any vectors left after applying the pre/post
@@ -188,15 +244,9 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) 
	if (nvecs == affd->pre_vectors + affd->post_vectors)

		return NULL;



	if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))

		return NULL;



	if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL))

		goto outcpumsk;



	node_to_cpumask = alloc_node_to_cpumask();

	if (!node_to_cpumask)

		goto outnpresmsk;

		return NULL;



	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);

	if (!masks)
@@ -206,30 +256,28 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) 
	for (curvec = 0; curvec < affd->pre_vectors; curvec++)

		cpumask_copy(masks + curvec, irq_default_affinity);



	/* Stabilize the cpumasks */

	get_online_cpus();

	build_node_to_cpumask(node_to_cpumask);



	/* Spread on present CPUs starting from affd->pre_vectors */

	usedvecs = irq_build_affinity_masks(affd, curvec, affvecs,

					    node_to_cpumask, cpu_present_mask,

					    nmsk, masks);



	/*

	 * Spread on non present CPUs starting from the next vector to be

	 * handled. If the spreading of present CPUs already exhausted the

	 * vector space, assign the non present CPUs to the already spread

	 * out vectors.

	 * Spread on present CPUs starting from affd->pre_vectors. If we

	 * have multiple sets, build each sets affinity mask separately.

	 */

	if (usedvecs >= affvecs)

		curvec = affd->pre_vectors;

	else

		curvec = affd->pre_vectors + usedvecs;

	cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask);

	usedvecs += irq_build_affinity_masks(affd, curvec, affvecs,

					     node_to_cpumask, npresmsk,

					     nmsk, masks);

	put_online_cpus();

	nr_sets = affd->nr_sets;

	if (!nr_sets)

		nr_sets = 1;



	for (i = 0, usedvecs = 0; i < nr_sets; i++) {

		int this_vecs = affd->sets ? affd->sets[i] : affvecs;

		int ret;



		ret = irq_build_affinity_masks(affd, curvec, this_vecs,

						curvec, node_to_cpumask, masks);

		if (ret) {

				kfree(masks);

				masks = NULL;

				goto outnodemsk;

		}

		curvec += this_vecs;

		usedvecs += this_vecs;

	}



	/* Fill out vectors at the end that don't need affinity */

	if (usedvecs >= affvecs)
@@ -241,10 +289,6 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) 


outnodemsk:

	free_node_to_cpumask(node_to_cpumask);

outnpresmsk:

	free_cpumask_var(npresmsk);

outcpumsk:

	free_cpumask_var(nmsk);

	return masks;

}
@@ -258,13 +302,21 @@ int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity 
{

	int resv = affd->pre_vectors + affd->post_vectors;

	int vecs = maxvec - resv;

	int ret;

	int set_vecs;



	if (resv > minvec)

		return 0;



	if (affd->nr_sets) {

		int i;



		for (i = 0, set_vecs = 0;  i < affd->nr_sets; i++)

			set_vecs += affd->sets[i];

	} else {

		get_online_cpus();

	ret = min_t(int, cpumask_weight(cpu_possible_mask), vecs) + resv;

		set_vecs = cpumask_weight(cpu_possible_mask);

		put_online_cpus();

	return ret;

	}



	return resv + min(set_vecs, vecs);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE