Commit 222e9239 authored by James Smart's avatar James Smart Committed by Martin K. Petersen
Browse files

scsi: lpfc: Resize cpu maps structures based on possible cpus 



The work done to date utilized the number of present cpus when sizing
per-cpu structures. Structures should have been sized based on the max
possible cpu count.

Convert the driver over to possible cpu count for sizing allocation.

Signed-off-by: default avatarDick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: default avatarJames Smart <jsmart2021@gmail.com>
Reviewed-by: default avatarHannes Reinecke <hare@suse.com>
Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parent 75508a8b

drivers/scsi/lpfc/lpfc_attr.c

+15 −8
Original line number Diff line number Diff line
@@ -5176,16 +5176,22 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr, 
	case 1:

		len += snprintf(buf + len, PAGE_SIZE-len,

				"fcp_cpu_map: HBA centric mapping (%d): "

				"%d online CPUs\n",

				phba->cfg_fcp_cpu_map,

				phba->sli4_hba.num_online_cpu);

				"%d of %d CPUs online from %d possible CPUs\n",

				phba->cfg_fcp_cpu_map, num_online_cpus(),

				num_present_cpus(),

				phba->sli4_hba.num_possible_cpu);

		break;

	}



	while (phba->sli4_hba.curr_disp_cpu < phba->sli4_hba.num_present_cpu) {

	while (phba->sli4_hba.curr_disp_cpu <

	       phba->sli4_hba.num_possible_cpu) {

		cpup = &phba->sli4_hba.cpu_map[phba->sli4_hba.curr_disp_cpu];



		if (cpup->irq == LPFC_VECTOR_MAP_EMPTY) {

		if (!cpu_present(phba->sli4_hba.curr_disp_cpu))

			len += snprintf(buf + len, PAGE_SIZE - len,

					"CPU %02d not present\n",

					phba->sli4_hba.curr_disp_cpu);

		else if (cpup->irq == LPFC_VECTOR_MAP_EMPTY) {

			if (cpup->hdwq == LPFC_VECTOR_MAP_EMPTY)

				len += snprintf(

					buf + len, PAGE_SIZE - len,
@@ -5225,14 +5231,15 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr, 


		/* display max number of CPUs keeping some margin */

		if (phba->sli4_hba.curr_disp_cpu <

				phba->sli4_hba.num_present_cpu &&

				phba->sli4_hba.num_possible_cpu &&

				(len >= (PAGE_SIZE - 64))) {

			len += snprintf(buf + len, PAGE_SIZE-len, "more...\n");

			len += snprintf(buf + len,

					PAGE_SIZE - len, "more...\n");

			break;

		}

	}



	if (phba->sli4_hba.curr_disp_cpu == phba->sli4_hba.num_present_cpu)

	if (phba->sli4_hba.curr_disp_cpu == phba->sli4_hba.num_possible_cpu)

		phba->sli4_hba.curr_disp_cpu = 0;



	return len;

drivers/scsi/lpfc/lpfc_init.c

+13 −19
Original line number Diff line number Diff line
@@ -6373,8 +6373,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) 
	u32 if_type;

	u32 if_fam;



	phba->sli4_hba.num_online_cpu = num_online_cpus();

	phba->sli4_hba.num_present_cpu = lpfc_present_cpu;

	phba->sli4_hba.num_possible_cpu = num_possible_cpus();

	phba->sli4_hba.curr_disp_cpu = 0;



	/* Get all the module params for configuring this host */
@@ -6796,7 +6796,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) 
		goto out_free_fcf_rr_bmask;

	}



	phba->sli4_hba.cpu_map = kcalloc(phba->sli4_hba.num_present_cpu,

	phba->sli4_hba.cpu_map = kcalloc(phba->sli4_hba.num_possible_cpu,

					sizeof(struct lpfc_vector_map_info),

					GFP_KERNEL);

	if (!phba->sli4_hba.cpu_map) {
@@ -6868,8 +6868,8 @@ lpfc_sli4_driver_resource_unset(struct lpfc_hba *phba) 


	/* Free memory allocated for msi-x interrupt vector to CPU mapping */

	kfree(phba->sli4_hba.cpu_map);

	phba->sli4_hba.num_possible_cpu = 0;

	phba->sli4_hba.num_present_cpu = 0;

	phba->sli4_hba.num_online_cpu = 0;

	phba->sli4_hba.curr_disp_cpu = 0;



	/* Free memory allocated for fast-path work queue handles */
@@ -10519,15 +10519,14 @@ lpfc_find_cpu_handle(struct lpfc_hba *phba, uint16_t id, int match) 
	int cpu;



	/* Find the desired phys_id for the specified EQ */

	cpup = phba->sli4_hba.cpu_map;

	for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {

	for_each_present_cpu(cpu) {

		cpup = &phba->sli4_hba.cpu_map[cpu];

		if ((match == LPFC_FIND_BY_EQ) &&

		    (cpup->irq != LPFC_VECTOR_MAP_EMPTY) &&

		    (cpup->eq == id))

			return cpu;

		if ((match == LPFC_FIND_BY_HDWQ) && (cpup->hdwq == id))

			return cpu;

		cpup++;

	}

	return 0;

}
@@ -10545,11 +10544,10 @@ lpfc_find_eq_handle(struct lpfc_hba *phba, uint16_t hdwq) 
	int cpu;



	/* Find the desired phys_id for the specified EQ */

	cpup = phba->sli4_hba.cpu_map;

	for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {

	for_each_present_cpu(cpu) {

		cpup = &phba->sli4_hba.cpu_map[cpu];

		if (cpup->hdwq == hdwq)

			return cpup->eq;

		cpup++;

	}

	return 0;

}
@@ -10569,16 +10567,14 @@ lpfc_find_hyper(struct lpfc_hba *phba, int cpu, 
	struct lpfc_vector_map_info *cpup;

	int idx;



	cpup = phba->sli4_hba.cpu_map;

	for (idx = 0; idx < phba->sli4_hba.num_present_cpu; idx++) {

	for_each_present_cpu(idx) {

		cpup = &phba->sli4_hba.cpu_map[idx];

		/* Does the cpup match the one we are looking for */

		if ((cpup->phys_id == phys_id) &&

		    (cpup->core_id == core_id) &&

		    (cpu != idx)) {

		    (cpu != idx))

			return 1;

	}

		cpup++;

	}

	return 0;

}

#endif
@@ -10608,7 +10604,7 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) 
	/* Init cpu_map array */

	memset(phba->sli4_hba.cpu_map, 0xff,

	       (sizeof(struct lpfc_vector_map_info) *

	       phba->sli4_hba.num_present_cpu));

	       phba->sli4_hba.num_possible_cpu));



	max_phys_id = 0;

	min_phys_id = 0xffff;
@@ -10617,8 +10613,8 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) 
	phys_id = 0;



	/* Update CPU map with physical id and core id of each CPU */

	cpup = phba->sli4_hba.cpu_map;

	for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {

	for_each_present_cpu(cpu) {

		cpup = &phba->sli4_hba.cpu_map[cpu];

#ifdef CONFIG_X86

		cpuinfo = &cpu_data(cpu);

		cpup->phys_id = cpuinfo->phys_proc_id;
@@ -10645,8 +10641,6 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) 
			max_core_id = cpup->core_id;

		if (cpup->core_id < min_core_id)

			min_core_id = cpup->core_id;



		cpup++;

	}



	for_each_possible_cpu(i) {

drivers/scsi/lpfc/lpfc_nvmet.c

+22 −13
Original line number Diff line number Diff line
@@ -1194,9 +1194,9 @@ lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba) 


	/* Cycle the the entire CPU context list for every MRQ */

	for (i = 0; i < phba->cfg_nvmet_mrq; i++) {

		for (j = 0; j < phba->sli4_hba.num_present_cpu; j++) {

		for_each_present_cpu(j) {

			infop = lpfc_get_ctx_list(phba, j, i);

			__lpfc_nvmet_clean_io_for_cpu(phba, infop);

			infop++; /* next */

		}

	}

	kfree(phba->sli4_hba.nvmet_ctx_info);
@@ -1211,14 +1211,14 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 
	union lpfc_wqe128 *wqe;

	struct lpfc_nvmet_ctx_info *last_infop;

	struct lpfc_nvmet_ctx_info *infop;

	int i, j, idx;

	int i, j, idx, cpu;



	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,

			"6403 Allocate NVMET resources for %d XRIs\n",

			phba->sli4_hba.nvmet_xri_cnt);



	phba->sli4_hba.nvmet_ctx_info = kcalloc(

		phba->sli4_hba.num_present_cpu * phba->cfg_nvmet_mrq,

		phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq,

		sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);

	if (!phba->sli4_hba.nvmet_ctx_info) {

		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
@@ -1246,13 +1246,12 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 
	 * of the IO completion. Thus a context that was allocated for MRQ A

	 * whose IO completed on CPU B will be freed to cpuB/mrqA.

	 */

	infop = phba->sli4_hba.nvmet_ctx_info;

	for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {

	for_each_possible_cpu(i) {

		for (j = 0; j < phba->cfg_nvmet_mrq; j++) {

			infop = lpfc_get_ctx_list(phba, i, j);

			INIT_LIST_HEAD(&infop->nvmet_ctx_list);

			spin_lock_init(&infop->nvmet_ctx_list_lock);

			infop->nvmet_ctx_list_cnt = 0;

			infop++;

		}

	}
@@ -1262,8 +1261,10 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 
	 * MRQ 1 cycling thru CPUs 0 - X, and so on.

	 */

	for (j = 0; j < phba->cfg_nvmet_mrq; j++) {

		last_infop = lpfc_get_ctx_list(phba, 0, j);

		for (i = phba->sli4_hba.num_present_cpu - 1;  i >= 0; i--) {

		last_infop = lpfc_get_ctx_list(phba,

					       cpumask_first(cpu_present_mask),

					       j);

		for (i = phba->sli4_hba.num_possible_cpu - 1;  i >= 0; i--) {

			infop = lpfc_get_ctx_list(phba, i, j);

			infop->nvmet_ctx_next_cpu = last_infop;

			last_infop = infop;
@@ -1274,6 +1275,7 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 
	 * received command on a per xri basis.

	 */

	idx = 0;

	cpu = cpumask_first(cpu_present_mask);

	for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {

		ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);

		if (!ctx_buf) {
@@ -1327,7 +1329,7 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 
		 * is MRQidx will be associated with CPUidx. This association

		 * can change on the fly.

		 */

		infop = lpfc_get_ctx_list(phba, idx, idx);

		infop = lpfc_get_ctx_list(phba, cpu, idx);

		spin_lock(&infop->nvmet_ctx_list_lock);

		list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);

		infop->nvmet_ctx_list_cnt++;
@@ -1335,11 +1337,18 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 


		/* Spread ctx structures evenly across all MRQs */

		idx++;

		if (idx >= phba->cfg_nvmet_mrq)

		if (idx >= phba->cfg_nvmet_mrq) {

			idx = 0;

			cpu = cpumask_first(cpu_present_mask);

			continue;

		}

		cpu = cpumask_next(cpu, cpu_present_mask);

		if (cpu == nr_cpu_ids)

			cpu = cpumask_first(cpu_present_mask);



	}



	for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {

	for_each_present_cpu(i) {

		for (j = 0; j < phba->cfg_nvmet_mrq; j++) {

			infop = lpfc_get_ctx_list(phba, i, j);

			lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
@@ -1839,7 +1848,7 @@ lpfc_nvmet_replenish_context(struct lpfc_hba *phba, 
	else

		get_infop = current_infop->nvmet_ctx_next_cpu;



	for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {

	for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) {

		if (get_infop == current_infop) {

			get_infop = get_infop->nvmet_ctx_next_cpu;

			continue;

drivers/scsi/lpfc/lpfc_sli4.h

+1 −1
Original line number Diff line number Diff line
@@ -890,7 +890,7 @@ struct lpfc_sli4_hba { 


	/* CPU to vector mapping information */

	struct lpfc_vector_map_info *cpu_map;

	uint16_t num_online_cpu;

	uint16_t num_possible_cpu;

	uint16_t num_present_cpu;

	uint16_t curr_disp_cpu;

	struct lpfc_eq_intr_info __percpu *eq_info;

CRA Git | Maintained and supported by SUSTech CRA and CCSE