Commit bf3b1e0c authored by Michael J. Ruhl's avatar Michael J. Ruhl Committed by Jason Gunthorpe
Browse files

IB/hfi1: Reduce excessive aspm inlines 



Uninline the aspm API since it increases code space for no reason.

Move the aspm module param to the new aspm C file.

Reviewed-by: default avatarMike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: default avatarMichael J. Ruhl <michael.j.ruhl@intel.com>
Signed-off-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent 2b0ad2da

drivers/infiniband/hw/hfi1/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ obj-$(CONFIG_INFINIBAND_HFI1) += hfi1.o 


hfi1-y := \

	affinity.o \

	aspm.o \

	chip.o \

	device.o \

	driver.o \

drivers/infiniband/hw/hfi1/aspm.c

0 → 100644
+270 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)

/*

 * Copyright(c) 2019 Intel Corporation.

 *

 */



#include "aspm.h"



/* Time after which the timer interrupt will re-enable ASPM */

#define ASPM_TIMER_MS 1000

/* Time for which interrupts are ignored after a timer has been scheduled */

#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)

/* Two interrupts within this time trigger ASPM disable */

#define ASPM_TRIGGER_MS 1

#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)

#define ASPM_L1_SUPPORTED(reg) \

	((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)



uint aspm_mode = ASPM_MODE_DISABLED;

module_param_named(aspm, aspm_mode, uint, 0444);

MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");



static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)

{

	struct pci_dev *parent = dd->pcidev->bus->self;

	u32 up, dn;



	/*

	 * If the driver does not have access to the upstream component,

	 * it cannot support ASPM L1 at all.

	 */

	if (!parent)

		return false;



	pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);

	dn = ASPM_L1_SUPPORTED(dn);



	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);

	up = ASPM_L1_SUPPORTED(up);



	/* ASPM works on A-step but is reported as not supported */

	return (!!dn || is_ax(dd)) && !!up;

}



/* Set L1 entrance latency for slower entry to L1 */

static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)

{

	u32 l1_ent_lat = 0x4u;

	u32 reg32;



	pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, &reg32);

	reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;

	reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;

	pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);

}



static void aspm_hw_enable_l1(struct hfi1_devdata *dd)

{

	struct pci_dev *parent = dd->pcidev->bus->self;



	/*

	 * If the driver does not have access to the upstream component,

	 * it cannot support ASPM L1 at all.

	 */

	if (!parent)

		return;



	/* Enable ASPM L1 first in upstream component and then downstream */

	pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,

					   PCI_EXP_LNKCTL_ASPMC,

					   PCI_EXP_LNKCTL_ASPM_L1);

	pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,

					   PCI_EXP_LNKCTL_ASPMC,

					   PCI_EXP_LNKCTL_ASPM_L1);

}



void aspm_hw_disable_l1(struct hfi1_devdata *dd)

{

	struct pci_dev *parent = dd->pcidev->bus->self;



	/* Disable ASPM L1 first in downstream component and then upstream */

	pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,

					   PCI_EXP_LNKCTL_ASPMC, 0x0);

	if (parent)

		pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,

						   PCI_EXP_LNKCTL_ASPMC, 0x0);

}



static  void aspm_enable(struct hfi1_devdata *dd)

{

	if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||

	    !dd->aspm_supported)

		return;



	aspm_hw_enable_l1(dd);

	dd->aspm_enabled = true;

}



static  void aspm_disable(struct hfi1_devdata *dd)

{

	if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)

		return;



	aspm_hw_disable_l1(dd);

	dd->aspm_enabled = false;

}



static  void aspm_disable_inc(struct hfi1_devdata *dd)

{

	unsigned long flags;



	spin_lock_irqsave(&dd->aspm_lock, flags);

	aspm_disable(dd);

	atomic_inc(&dd->aspm_disabled_cnt);

	spin_unlock_irqrestore(&dd->aspm_lock, flags);

}



static  void aspm_enable_dec(struct hfi1_devdata *dd)

{

	unsigned long flags;



	spin_lock_irqsave(&dd->aspm_lock, flags);

	if (atomic_dec_and_test(&dd->aspm_disabled_cnt))

		aspm_enable(dd);

	spin_unlock_irqrestore(&dd->aspm_lock, flags);

}



/* ASPM processing for each receive context interrupt */

void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)

{

	bool restart_timer;

	bool close_interrupts;

	unsigned long flags;

	ktime_t now, prev;



	spin_lock_irqsave(&rcd->aspm_lock, flags);

	/* PSM contexts are open */

	if (!rcd->aspm_intr_enable)

		goto unlock;



	prev = rcd->aspm_ts_last_intr;

	now = ktime_get();

	rcd->aspm_ts_last_intr = now;



	/* An interrupt pair close together in time */

	close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;



	/* Don't push out our timer till this much time has elapsed */

	restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >

				    ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;

	restart_timer = restart_timer && close_interrupts;



	/* Disable ASPM and schedule timer */

	if (rcd->aspm_enabled && close_interrupts) {

		aspm_disable_inc(rcd->dd);

		rcd->aspm_enabled = false;

		restart_timer = true;

	}



	if (restart_timer) {

		mod_timer(&rcd->aspm_timer,

			  jiffies + msecs_to_jiffies(ASPM_TIMER_MS));

		rcd->aspm_ts_timer_sched = now;

	}

unlock:

	spin_unlock_irqrestore(&rcd->aspm_lock, flags);

}



/* Timer function for re-enabling ASPM in the absence of interrupt activity */

static  void aspm_ctx_timer_function(struct timer_list *t)

{

	struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);

	unsigned long flags;



	spin_lock_irqsave(&rcd->aspm_lock, flags);

	aspm_enable_dec(rcd->dd);

	rcd->aspm_enabled = true;

	spin_unlock_irqrestore(&rcd->aspm_lock, flags);

}



/*

 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts

 * are open.

 */

void aspm_disable_all(struct hfi1_devdata *dd)

{

	struct hfi1_ctxtdata *rcd;

	unsigned long flags;

	u16 i;



	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {

		rcd = hfi1_rcd_get_by_index(dd, i);

		if (rcd) {

			del_timer_sync(&rcd->aspm_timer);

			spin_lock_irqsave(&rcd->aspm_lock, flags);

			rcd->aspm_intr_enable = false;

			spin_unlock_irqrestore(&rcd->aspm_lock, flags);

			hfi1_rcd_put(rcd);

		}

	}



	aspm_disable(dd);

	atomic_set(&dd->aspm_disabled_cnt, 0);

}



/* Re-enable interrupt processing for verbs contexts */

void aspm_enable_all(struct hfi1_devdata *dd)

{

	struct hfi1_ctxtdata *rcd;

	unsigned long flags;

	u16 i;



	aspm_enable(dd);



	if (aspm_mode != ASPM_MODE_DYNAMIC)

		return;



	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {

		rcd = hfi1_rcd_get_by_index(dd, i);

		if (rcd) {

			spin_lock_irqsave(&rcd->aspm_lock, flags);

			rcd->aspm_intr_enable = true;

			rcd->aspm_enabled = true;

			spin_unlock_irqrestore(&rcd->aspm_lock, flags);

			hfi1_rcd_put(rcd);

		}

	}

}



static  void aspm_ctx_init(struct hfi1_ctxtdata *rcd)

{

	spin_lock_init(&rcd->aspm_lock);

	timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);

	rcd->aspm_intr_supported = rcd->dd->aspm_supported &&

		aspm_mode == ASPM_MODE_DYNAMIC &&

		rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;

}



void aspm_init(struct hfi1_devdata *dd)

{

	struct hfi1_ctxtdata *rcd;

	u16 i;



	spin_lock_init(&dd->aspm_lock);

	dd->aspm_supported = aspm_hw_l1_supported(dd);



	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {

		rcd = hfi1_rcd_get_by_index(dd, i);

		if (rcd)

			aspm_ctx_init(rcd);

		hfi1_rcd_put(rcd);

	}



	/* Start with ASPM disabled */

	aspm_hw_set_l1_ent_latency(dd);

	dd->aspm_enabled = false;

	aspm_hw_disable_l1(dd);



	/* Now turn on ASPM if configured */

	aspm_enable_all(dd);

}



void aspm_exit(struct hfi1_devdata *dd)

{

	aspm_disable_all(dd);



	/* Turn on ASPM on exit to conserve power */

	aspm_enable(dd);

}

drivers/infiniband/hw/hfi1/aspm.h

+8 −254
Original line number Diff line number Diff line
@@ -57,266 +57,20 @@ enum aspm_mode { 
	ASPM_MODE_DYNAMIC = 2,	/* ASPM enabled/disabled dynamically */

};



/* Time after which the timer interrupt will re-enable ASPM */

#define ASPM_TIMER_MS 1000

/* Time for which interrupts are ignored after a timer has been scheduled */

#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)

/* Two interrupts within this time trigger ASPM disable */

#define ASPM_TRIGGER_MS 1

#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)

#define ASPM_L1_SUPPORTED(reg) \

	(((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)

void aspm_init(struct hfi1_devdata *dd);

void aspm_exit(struct hfi1_devdata *dd);

void aspm_hw_disable_l1(struct hfi1_devdata *dd);

void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd);

void aspm_disable_all(struct hfi1_devdata *dd);

void aspm_enable_all(struct hfi1_devdata *dd);



static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd)

{

	struct pci_dev *parent = dd->pcidev->bus->self;

	u32 up, dn;



	/*

	 * If the driver does not have access to the upstream component,

	 * it cannot support ASPM L1 at all.

	 */

	if (!parent)

		return false;



	pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);

	dn = ASPM_L1_SUPPORTED(dn);



	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);

	up = ASPM_L1_SUPPORTED(up);



	/* ASPM works on A-step but is reported as not supported */

	return (!!dn || is_ax(dd)) && !!up;

}



/* Set L1 entrance latency for slower entry to L1 */

static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)

{

	u32 l1_ent_lat = 0x4u;

	u32 reg32;



	pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, &reg32);

	reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;

	reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;

	pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);

}



static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd)

{

	struct pci_dev *parent = dd->pcidev->bus->self;



	/*

	 * If the driver does not have access to the upstream component,

	 * it cannot support ASPM L1 at all.

	 */

	if (!parent)

		return;



	/* Enable ASPM L1 first in upstream component and then downstream */

	pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,

					   PCI_EXP_LNKCTL_ASPMC,

					   PCI_EXP_LNKCTL_ASPM_L1);

	pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,

					   PCI_EXP_LNKCTL_ASPMC,

					   PCI_EXP_LNKCTL_ASPM_L1);

}



static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd)

{

	struct pci_dev *parent = dd->pcidev->bus->self;



	/* Disable ASPM L1 first in downstream component and then upstream */

	pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,

					   PCI_EXP_LNKCTL_ASPMC, 0x0);

	if (parent)

		pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,

						   PCI_EXP_LNKCTL_ASPMC, 0x0);

}



static inline void aspm_enable(struct hfi1_devdata *dd)

{

	if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||

	    !dd->aspm_supported)

		return;



	aspm_hw_enable_l1(dd);

	dd->aspm_enabled = true;

}



static inline void aspm_disable(struct hfi1_devdata *dd)

{

	if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)

		return;



	aspm_hw_disable_l1(dd);

	dd->aspm_enabled = false;

}



static inline void aspm_disable_inc(struct hfi1_devdata *dd)

{

	unsigned long flags;



	spin_lock_irqsave(&dd->aspm_lock, flags);

	aspm_disable(dd);

	atomic_inc(&dd->aspm_disabled_cnt);

	spin_unlock_irqrestore(&dd->aspm_lock, flags);

}



static inline void aspm_enable_dec(struct hfi1_devdata *dd)

{

	unsigned long flags;



	spin_lock_irqsave(&dd->aspm_lock, flags);

	if (atomic_dec_and_test(&dd->aspm_disabled_cnt))

		aspm_enable(dd);

	spin_unlock_irqrestore(&dd->aspm_lock, flags);

}



/* ASPM processing for each receive context interrupt */

static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd)

{

	bool restart_timer;

	bool close_interrupts;

	unsigned long flags;

	ktime_t now, prev;



	/* Quickest exit for minimum impact */

	if (!rcd->aspm_intr_supported)

	if (likely(!rcd->aspm_intr_supported))

		return;



	spin_lock_irqsave(&rcd->aspm_lock, flags);

	/* PSM contexts are open */

	if (!rcd->aspm_intr_enable)

		goto unlock;



	prev = rcd->aspm_ts_last_intr;

	now = ktime_get();

	rcd->aspm_ts_last_intr = now;



	/* An interrupt pair close together in time */

	close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;



	/* Don't push out our timer till this much time has elapsed */

	restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >

				    ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;

	restart_timer = restart_timer && close_interrupts;



	/* Disable ASPM and schedule timer */

	if (rcd->aspm_enabled && close_interrupts) {

		aspm_disable_inc(rcd->dd);

		rcd->aspm_enabled = false;

		restart_timer = true;

	}



	if (restart_timer) {

		mod_timer(&rcd->aspm_timer,

			  jiffies + msecs_to_jiffies(ASPM_TIMER_MS));

		rcd->aspm_ts_timer_sched = now;

	}

unlock:

	spin_unlock_irqrestore(&rcd->aspm_lock, flags);

}



/* Timer function for re-enabling ASPM in the absence of interrupt activity */

static inline void aspm_ctx_timer_function(struct timer_list *t)

{

	struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);

	unsigned long flags;



	spin_lock_irqsave(&rcd->aspm_lock, flags);

	aspm_enable_dec(rcd->dd);

	rcd->aspm_enabled = true;

	spin_unlock_irqrestore(&rcd->aspm_lock, flags);

}



/*

 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts

 * are open.

 */

static inline void aspm_disable_all(struct hfi1_devdata *dd)

{

	struct hfi1_ctxtdata *rcd;

	unsigned long flags;

	u16 i;



	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {

		rcd = hfi1_rcd_get_by_index(dd, i);

		if (rcd) {

			del_timer_sync(&rcd->aspm_timer);

			spin_lock_irqsave(&rcd->aspm_lock, flags);

			rcd->aspm_intr_enable = false;

			spin_unlock_irqrestore(&rcd->aspm_lock, flags);

			hfi1_rcd_put(rcd);

		}

	}



	aspm_disable(dd);

	atomic_set(&dd->aspm_disabled_cnt, 0);

}



/* Re-enable interrupt processing for verbs contexts */

static inline void aspm_enable_all(struct hfi1_devdata *dd)

{

	struct hfi1_ctxtdata *rcd;

	unsigned long flags;

	u16 i;



	aspm_enable(dd);



	if (aspm_mode != ASPM_MODE_DYNAMIC)

		return;



	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {

		rcd = hfi1_rcd_get_by_index(dd, i);

		if (rcd) {

			spin_lock_irqsave(&rcd->aspm_lock, flags);

			rcd->aspm_intr_enable = true;

			rcd->aspm_enabled = true;

			spin_unlock_irqrestore(&rcd->aspm_lock, flags);

			hfi1_rcd_put(rcd);

		}

	}

}



static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd)

{

	spin_lock_init(&rcd->aspm_lock);

	timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);

	rcd->aspm_intr_supported = rcd->dd->aspm_supported &&

		aspm_mode == ASPM_MODE_DYNAMIC &&

		rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;

}



static inline void aspm_init(struct hfi1_devdata *dd)

{

	struct hfi1_ctxtdata *rcd;

	u16 i;



	spin_lock_init(&dd->aspm_lock);

	dd->aspm_supported = aspm_hw_l1_supported(dd);



	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {

		rcd = hfi1_rcd_get_by_index(dd, i);

		if (rcd)

			aspm_ctx_init(rcd);

		hfi1_rcd_put(rcd);

	}



	/* Start with ASPM disabled */

	aspm_hw_set_l1_ent_latency(dd);

	dd->aspm_enabled = false;

	aspm_hw_disable_l1(dd);



	/* Now turn on ASPM if configured */

	aspm_enable_all(dd);

}



static inline void aspm_exit(struct hfi1_devdata *dd)

{

	aspm_disable_all(dd);



	/* Turn on ASPM on exit to conserve power */

	aspm_enable(dd);

	__aspm_ctx_disable(rcd);

}



#endif /* _ASPM_H */

drivers/infiniband/hw/hfi1/pcie.c

+1 −5
Original line number Diff line number Diff line 
/*

 * Copyright(c) 2015 - 2018 Intel Corporation.

 * Copyright(c) 2015 - 2019 Intel Corporation.

 *

 * This file is provided under a dual BSD/GPLv2 license.  When using or

 * redistributing this file, you may do so under either license.
@@ -450,10 +450,6 @@ static int hfi1_pcie_caps; 
module_param_named(pcie_caps, hfi1_pcie_caps, int, 0444);

MODULE_PARM_DESC(pcie_caps, "Max PCIe tuning: Payload (0..3), ReadReq (4..7)");



uint aspm_mode = ASPM_MODE_DISABLED;

module_param_named(aspm, aspm_mode, uint, 0444);

MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");



/**

 * tune_pcie_caps() - Code to adjust PCIe capabilities.

 * @dd: Valid device data structure

CRA Git | Maintained and supported by SUSTech CRA and CCSE