Commit e9dfbaae authored by Huacai Chen's avatar Huacai Chen Committed by Thomas Bogendoerfer
Browse files

MIPS: perf: Add hardware perf events support for new Loongson-3 



New Loongson-3 means Loongson-3A R2 (Loongson-3A2000) and newer CPUs.
Loongson-3 processors have three types of PMU types (so there are three
event maps): Loongson-3A1000/Loonngson-3B1000/Loongson-3B1500 is Type-1,
Loongson-3A2000/Loongson-3A3000 is Type-2, Loongson-3A4000+ is Type-3.

Signed-off-by: default avatarHuacai Chen <chenhc@lemote.com>
Signed-off-by: default avatarThomas Bogendoerfer <tsbogend@alpha.franken.de>
parent 44220fd8

arch/mips/include/asm/mach-loongson64/cpu-feature-overrides.h

+1 −0
Original line number Diff line number Diff line
@@ -48,5 +48,6 @@ 
#define cpu_hwrena_impl_bits	0xc0000000

#define cpu_has_mac2008_only	1

#define cpu_has_mips_r2_exec_hazard 0

#define cpu_has_perf_cntr_intr_bit	0



#endif /* __ASM_MACH_LOONGSON64_CPU_FEATURE_OVERRIDES_H */

arch/mips/kernel/perf_event_mipsxx.c

+340 −33
Original line number Diff line number Diff line
@@ -90,6 +90,7 @@ struct mips_pmu { 
	unsigned int	num_counters;

};



static int counter_bits;

static struct mips_pmu mipspmu;



#define M_PERFCTL_EVENT(event)		(((event) << MIPS_PERFCTRL_EVENT_S) & \
@@ -118,6 +119,7 @@ static struct mips_pmu mipspmu; 
#define M_PERFCTL_CONFIG_MASK		0x1f

#endif



#define CNTR_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))



#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS

static DEFINE_RWLOCK(pmuint_rwlock);
@@ -154,6 +156,31 @@ static void pause_local_counters(void); 
static irqreturn_t mipsxx_pmu_handle_irq(int, void *);

static int mipsxx_pmu_handle_shared_irq(void);



/* 0: Not Loongson-3

 * 1: Loongson-3A1000/3B1000/3B1500

 * 2: Loongson-3A2000/3A3000

 * 3: Loongson-3A4000+

 */



#define LOONGSON_PMU_TYPE0 0

#define LOONGSON_PMU_TYPE1 1

#define LOONGSON_PMU_TYPE2 2

#define LOONGSON_PMU_TYPE3 3



static inline int get_loongson3_pmu_type(void)

{

	if (boot_cpu_type() != CPU_LOONGSON64)

		return LOONGSON_PMU_TYPE0;

	if ((boot_cpu_data.processor_id & PRID_COMP_MASK) == PRID_COMP_LEGACY)

		return LOONGSON_PMU_TYPE1;

	if ((boot_cpu_data.processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64C)

		return LOONGSON_PMU_TYPE2;

	if ((boot_cpu_data.processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64G)

		return LOONGSON_PMU_TYPE3;



	return LOONGSON_PMU_TYPE0;

}



static unsigned int mipsxx_pmu_swizzle_perf_idx(unsigned int idx)

{

	if (vpe_id() == 1)
@@ -186,17 +213,18 @@ static u64 mipsxx_pmu_read_counter(unsigned int idx) 


static u64 mipsxx_pmu_read_counter_64(unsigned int idx)

{

	u64 mask = CNTR_BIT_MASK(counter_bits);

	idx = mipsxx_pmu_swizzle_perf_idx(idx);



	switch (idx) {

	case 0:

		return read_c0_perfcntr0_64();

		return read_c0_perfcntr0_64() & mask;

	case 1:

		return read_c0_perfcntr1_64();

		return read_c0_perfcntr1_64() & mask;

	case 2:

		return read_c0_perfcntr2_64();

		return read_c0_perfcntr2_64() & mask;

	case 3:

		return read_c0_perfcntr3_64();

		return read_c0_perfcntr3_64() & mask;

	default:

		WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);

		return 0;
@@ -225,6 +253,7 @@ static void mipsxx_pmu_write_counter(unsigned int idx, u64 val) 


static void mipsxx_pmu_write_counter_64(unsigned int idx, u64 val)

{

	val &= CNTR_BIT_MASK(counter_bits);

	idx = mipsxx_pmu_swizzle_perf_idx(idx);



	switch (idx) {
@@ -286,12 +315,16 @@ static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, 
				    struct hw_perf_event *hwc)

{

	int i;

	unsigned long cntr_mask;



	/*

	 * We only need to care the counter mask. The range has been

	 * checked definitely.

	 */

	unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff;

	if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2)

		cntr_mask = (hwc->event_base >> 10) & 0xffff;

	else

		cntr_mask = (hwc->event_base >> 8) & 0xffff;



	for (i = mipspmu.num_counters - 1; i >= 0; i--) {

		/*
@@ -320,6 +353,12 @@ static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) 


	WARN_ON(idx < 0 || idx >= mipspmu.num_counters);



	if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2)

		cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0x3ff) |

			(evt->config_base & M_PERFCTL_CONFIG_MASK) |

			/* Make sure interrupt enabled. */

			MIPS_PERFCTRL_IE;

	else

		cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) |

			(evt->config_base & M_PERFCTL_CONFIG_MASK) |

			/* Make sure interrupt enabled. */
@@ -396,6 +435,10 @@ static int mipspmu_event_set_period(struct perf_event *event, 


	local64_set(&hwc->prev_count, mipspmu.overflow - left);



	if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2)

		mipsxx_pmu_write_control(idx,

				M_PERFCTL_EVENT(hwc->event_base & 0x3ff));



	mipspmu.write_counter(idx, mipspmu.overflow - left);



	perf_event_update_userpage(event);
@@ -667,8 +710,14 @@ static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) 
			(pev->event_id & 0xff);

	else

#endif /* CONFIG_MIPS_MT_SMP */

		return ((pev->cntr_mask & 0xffff00) |

			(pev->event_id & 0xff));

	{

		if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2)

			return (pev->cntr_mask & 0xfffc00) |

				(pev->event_id & 0x3ff);

		else

			return (pev->cntr_mask & 0xffff00) |

				(pev->event_id & 0xff);

	}

}



static const struct mips_perf_event *mipspmu_map_general_event(int idx)
@@ -783,26 +832,104 @@ static int n_counters(void) 
	return counters;

}



static void loongson3_reset_counters(void *arg)

{

	int counters = (int)(long)arg;



	switch (counters) {

	case 4:

		mipsxx_pmu_write_control(3, 0);

		mipspmu.write_counter(3, 0);

		mipsxx_pmu_write_control(3, 127<<5);

		mipspmu.write_counter(3, 0);

		mipsxx_pmu_write_control(3, 191<<5);

		mipspmu.write_counter(3, 0);

		mipsxx_pmu_write_control(3, 255<<5);

		mipspmu.write_counter(3, 0);

		mipsxx_pmu_write_control(3, 319<<5);

		mipspmu.write_counter(3, 0);

		mipsxx_pmu_write_control(3, 383<<5);

		mipspmu.write_counter(3, 0);

		mipsxx_pmu_write_control(3, 575<<5);

		mipspmu.write_counter(3, 0);

		fallthrough;

	case 3:

		mipsxx_pmu_write_control(2, 0);

		mipspmu.write_counter(2, 0);

		mipsxx_pmu_write_control(2, 127<<5);

		mipspmu.write_counter(2, 0);

		mipsxx_pmu_write_control(2, 191<<5);

		mipspmu.write_counter(2, 0);

		mipsxx_pmu_write_control(2, 255<<5);

		mipspmu.write_counter(2, 0);

		mipsxx_pmu_write_control(2, 319<<5);

		mipspmu.write_counter(2, 0);

		mipsxx_pmu_write_control(2, 383<<5);

		mipspmu.write_counter(2, 0);

		mipsxx_pmu_write_control(2, 575<<5);

		mipspmu.write_counter(2, 0);

		fallthrough;

	case 2:

		mipsxx_pmu_write_control(1, 0);

		mipspmu.write_counter(1, 0);

		mipsxx_pmu_write_control(1, 127<<5);

		mipspmu.write_counter(1, 0);

		mipsxx_pmu_write_control(1, 191<<5);

		mipspmu.write_counter(1, 0);

		mipsxx_pmu_write_control(1, 255<<5);

		mipspmu.write_counter(1, 0);

		mipsxx_pmu_write_control(1, 319<<5);

		mipspmu.write_counter(1, 0);

		mipsxx_pmu_write_control(1, 383<<5);

		mipspmu.write_counter(1, 0);

		mipsxx_pmu_write_control(1, 575<<5);

		mipspmu.write_counter(1, 0);

		fallthrough;

	case 1:

		mipsxx_pmu_write_control(0, 0);

		mipspmu.write_counter(0, 0);

		mipsxx_pmu_write_control(0, 127<<5);

		mipspmu.write_counter(0, 0);

		mipsxx_pmu_write_control(0, 191<<5);

		mipspmu.write_counter(0, 0);

		mipsxx_pmu_write_control(0, 255<<5);

		mipspmu.write_counter(0, 0);

		mipsxx_pmu_write_control(0, 319<<5);

		mipspmu.write_counter(0, 0);

		mipsxx_pmu_write_control(0, 383<<5);

		mipspmu.write_counter(0, 0);

		mipsxx_pmu_write_control(0, 575<<5);

		mipspmu.write_counter(0, 0);

		fallthrough;

	}

}



static void reset_counters(void *arg)

{

	int counters = (int)(long)arg;



	if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) {

		loongson3_reset_counters(arg);

		return;

	}



	switch (counters) {

	case 4:

		mipsxx_pmu_write_control(3, 0);

		mipspmu.write_counter(3, 0);

		/* fall through */

		fallthrough;

	case 3:

		mipsxx_pmu_write_control(2, 0);

		mipspmu.write_counter(2, 0);

		/* fall through */

		fallthrough;

	case 2:

		mipsxx_pmu_write_control(1, 0);

		mipspmu.write_counter(1, 0);

		/* fall through */

		fallthrough;

	case 1:

		mipsxx_pmu_write_control(0, 0);

		mipspmu.write_counter(0, 0);

		/* fall through */

		fallthrough;

	}

}
@@ -834,13 +961,30 @@ static const struct mips_perf_event i6x00_event_map[PERF_COUNT_HW_MAX] = { 
	[PERF_COUNT_HW_BRANCH_MISSES]       = { 0x16, CNTR_EVEN | CNTR_ODD },

};



static const struct mips_perf_event loongson3_event_map[PERF_COUNT_HW_MAX] = {

static const struct mips_perf_event loongson3_event_map1[PERF_COUNT_HW_MAX] = {

	[PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN },

	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, CNTR_ODD },

	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x01, CNTR_EVEN },

	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x01, CNTR_ODD },

};



static const struct mips_perf_event loongson3_event_map2[PERF_COUNT_HW_MAX] = {

	[PERF_COUNT_HW_CPU_CYCLES] = { 0x80, CNTR_ALL },

	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x81, CNTR_ALL },

	[PERF_COUNT_HW_CACHE_MISSES] = { 0x18, CNTR_ALL },

	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x94, CNTR_ALL },

	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x9c, CNTR_ALL },

};



static const struct mips_perf_event loongson3_event_map3[PERF_COUNT_HW_MAX] = {

	[PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_ALL },

	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_ALL },

	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x1c, CNTR_ALL },

	[PERF_COUNT_HW_CACHE_MISSES] = { 0x1d, CNTR_ALL },

	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_ALL },

	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x08, CNTR_ALL },

};



static const struct mips_perf_event octeon_event_map[PERF_COUNT_HW_MAX] = {

	[PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL },

	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x03, CNTR_ALL },
@@ -1064,7 +1208,7 @@ static const struct mips_perf_event i6x00_cache_map 
},

};



static const struct mips_perf_event loongson3_cache_map

static const struct mips_perf_event loongson3_cache_map1

				[PERF_COUNT_HW_CACHE_MAX]

				[PERF_COUNT_HW_CACHE_OP_MAX]

				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
@@ -1109,12 +1253,127 @@ static const struct mips_perf_event loongson3_cache_map 
[C(BPU)] = {

	/* Using the same code for *HW_BRANCH* */

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN },

		[C(RESULT_MISS)]        = { 0x02, CNTR_ODD },

		[C(RESULT_ACCESS)]      = { 0x01, CNTR_EVEN },

		[C(RESULT_MISS)]        = { 0x01, CNTR_ODD },

	},

	[C(OP_WRITE)] = {

		[C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN },

		[C(RESULT_MISS)]        = { 0x02, CNTR_ODD },

		[C(RESULT_ACCESS)]      = { 0x01, CNTR_EVEN },

		[C(RESULT_MISS)]        = { 0x01, CNTR_ODD },

	},

},

};



static const struct mips_perf_event loongson3_cache_map2

				[PERF_COUNT_HW_CACHE_MAX]

				[PERF_COUNT_HW_CACHE_OP_MAX]

				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {

[C(L1D)] = {

	/*

	 * Like some other architectures (e.g. ARM), the performance

	 * counters don't differentiate between read and write

	 * accesses/misses, so this isn't strictly correct, but it's the

	 * best we can do. Writes and reads get combined.

	 */

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]	= { 0x156, CNTR_ALL },

	},

	[C(OP_WRITE)] = {

		[C(RESULT_ACCESS)]	= { 0x155, CNTR_ALL },

		[C(RESULT_MISS)]        = { 0x153, CNTR_ALL },

	},

},

[C(L1I)] = {

	[C(OP_READ)] = {

		[C(RESULT_MISS)]	= { 0x18, CNTR_ALL },

	},

	[C(OP_WRITE)] = {

		[C(RESULT_MISS)]        = { 0x18, CNTR_ALL },

	},

},

[C(LL)] = {

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]	= { 0x1b6, CNTR_ALL },

	},

	[C(OP_WRITE)] = {

		[C(RESULT_ACCESS)]	= { 0x1b7, CNTR_ALL },

	},

	[C(OP_PREFETCH)] = {

		[C(RESULT_ACCESS)]	= { 0x1bf, CNTR_ALL },

	},

},

[C(DTLB)] = {

	[C(OP_READ)] = {

		[C(RESULT_MISS)]        = { 0x92, CNTR_ALL },

	},

	[C(OP_WRITE)] = {

		[C(RESULT_MISS)]        = { 0x92, CNTR_ALL },

	},

},

[C(ITLB)] = {

	[C(OP_READ)] = {

		[C(RESULT_MISS)]	= { 0x1a, CNTR_ALL },

	},

	[C(OP_WRITE)] = {

		[C(RESULT_MISS)]	= { 0x1a, CNTR_ALL },

	},

},

[C(BPU)] = {

	/* Using the same code for *HW_BRANCH* */

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]      = { 0x94, CNTR_ALL },

		[C(RESULT_MISS)]        = { 0x9c, CNTR_ALL },

	},

},

};



static const struct mips_perf_event loongson3_cache_map3

				[PERF_COUNT_HW_CACHE_MAX]

				[PERF_COUNT_HW_CACHE_OP_MAX]

				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {

[C(L1D)] = {

	/*

	 * Like some other architectures (e.g. ARM), the performance

	 * counters don't differentiate between read and write

	 * accesses/misses, so this isn't strictly correct, but it's the

	 * best we can do. Writes and reads get combined.

	 */

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]      = { 0x1e, CNTR_ALL },

		[C(RESULT_MISS)]        = { 0x1f, CNTR_ALL },

	},

	[C(OP_PREFETCH)] = {

		[C(RESULT_ACCESS)]	= { 0xaa, CNTR_ALL },

		[C(RESULT_MISS)]	= { 0xa9, CNTR_ALL },

	},

},

[C(L1I)] = {

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]	= { 0x1c, CNTR_ALL },

		[C(RESULT_MISS)]	= { 0x1d, CNTR_ALL },

	},

},

[C(LL)] = {

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]	= { 0x2e, CNTR_ALL },

		[C(RESULT_MISS)]	= { 0x2f, CNTR_ALL },

	},

},

[C(DTLB)] = {

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]      = { 0x14, CNTR_ALL },

		[C(RESULT_MISS)]	= { 0x1b, CNTR_ALL },

	},

},

[C(ITLB)] = {

	[C(OP_READ)] = {

		[C(RESULT_MISS)]	= { 0x1a, CNTR_ALL },

	},

},

[C(BPU)] = {

	/* Using the same code for *HW_BRANCH* */

	[C(OP_READ)] = {

		[C(RESULT_ACCESS)]      = { 0x02, CNTR_ALL },

		[C(RESULT_MISS)]        = { 0x08, CNTR_ALL },

	},

},

};
@@ -1178,7 +1437,6 @@ static const struct mips_perf_event bmips5000_cache_map 
},

};





static const struct mips_perf_event octeon_cache_map

				[PERF_COUNT_HW_CACHE_MAX]

				[PERF_COUNT_HW_CACHE_OP_MAX]
@@ -1512,6 +1770,7 @@ static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev) 
static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config)

{

	/* currently most cores have 7-bit event numbers */

	int pmu_type;

	unsigned int raw_id = config & 0xff;

	unsigned int base_id = raw_id & 0x7f;
@@ -1624,7 +1883,32 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) 
				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;

		break;

	case CPU_LOONGSON64:

		raw_event.cntr_mask = raw_id > 127 ? CNTR_ODD : CNTR_EVEN;

		pmu_type = get_loongson3_pmu_type();



		switch (pmu_type) {

		case LOONGSON_PMU_TYPE1:

			raw_event.cntr_mask =

				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;

			break;

		case LOONGSON_PMU_TYPE2:

			base_id = config & 0x3ff;

			raw_event.cntr_mask = CNTR_ALL;



			if ((base_id >= 1 && base_id < 28) ||

				(base_id >= 64 && base_id < 90) ||

				(base_id >= 128 && base_id < 164) ||

				(base_id >= 192 && base_id < 200) ||

				(base_id >= 256 && base_id < 274) ||

				(base_id >= 320 && base_id < 358) ||

				(base_id >= 384 && base_id < 574))

				break;



			return ERR_PTR(-EOPNOTSUPP);

		case LOONGSON_PMU_TYPE3:

			base_id = raw_id;

			raw_event.cntr_mask = CNTR_ALL;

			break;

		}

		break;

	}
@@ -1683,8 +1967,7 @@ static const struct mips_perf_event *xlp_pmu_map_raw_event(u64 config) 
static int __init

init_hw_perf_events(void)

{

	int counters, irq;

	int counter_bits;

	int counters, irq, pmu_type;



	pr_info("Performance counters: ");
@@ -1771,8 +2054,25 @@ init_hw_perf_events(void) 
		break;

	case CPU_LOONGSON64:

		mipspmu.name = "mips/loongson3";

		mipspmu.general_event_map = &loongson3_event_map;

		mipspmu.cache_event_map = &loongson3_cache_map;

		pmu_type = get_loongson3_pmu_type();



		switch (pmu_type) {

		case LOONGSON_PMU_TYPE1:

			counters = 2;

			mipspmu.general_event_map = &loongson3_event_map1;

			mipspmu.cache_event_map = &loongson3_cache_map1;

			break;

		case LOONGSON_PMU_TYPE2:

			counters = 4;

			mipspmu.general_event_map = &loongson3_event_map2;

			mipspmu.cache_event_map = &loongson3_cache_map2;

			break;

		case LOONGSON_PMU_TYPE3:

			counters = 4;

			mipspmu.general_event_map = &loongson3_event_map3;

			mipspmu.cache_event_map = &loongson3_cache_map3;

			break;

		}

		break;

	case CPU_CAVIUM_OCTEON:

	case CPU_CAVIUM_OCTEON_PLUS:
@@ -1803,19 +2103,26 @@ init_hw_perf_events(void) 
	mipspmu.irq = irq;



	if (read_c0_perfctrl0() & MIPS_PERFCTRL_W) {

		if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) {

			counter_bits = 48;

			mipspmu.max_period = (1ULL << 47) - 1;

			mipspmu.valid_count = (1ULL << 47) - 1;

			mipspmu.overflow = 1ULL << 47;

		} else {

			counter_bits = 64;

			mipspmu.max_period = (1ULL << 63) - 1;

			mipspmu.valid_count = (1ULL << 63) - 1;

			mipspmu.overflow = 1ULL << 63;

		}

		mipspmu.read_counter = mipsxx_pmu_read_counter_64;

		mipspmu.write_counter = mipsxx_pmu_write_counter_64;

		counter_bits = 64;

	} else {

		counter_bits = 32;

		mipspmu.max_period = (1ULL << 31) - 1;

		mipspmu.valid_count = (1ULL << 31) - 1;

		mipspmu.overflow = 1ULL << 31;

		mipspmu.read_counter = mipsxx_pmu_read_counter;

		mipspmu.write_counter = mipsxx_pmu_write_counter;

		counter_bits = 32;

	}



	on_each_cpu(reset_counters, (void *)(long)counters, 1);

CRA Git | Maintained and supported by SUSTech CRA and CCSE