Commit 9dfa9a5c authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

perf/x86: Reduce stack usage for x86_pmu::drain_pebs() 



intel_pmu_drain_pebs_*() is typically called from handle_pmi_common(),
both have an on-stack struct perf_sample_data, which is *big*. Rewire
things so that drain_pebs() can use the one handle_pmi_common() has.

Reported-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20201030151955.054099690@infradead.org
parent 267fb273

arch/x86/events/intel/core.c

+1 −1
Original line number Diff line number Diff line
@@ -2630,7 +2630,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) 
		u64 pebs_enabled = cpuc->pebs_enabled;



		handled++;

		x86_pmu.drain_pebs(regs);

		x86_pmu.drain_pebs(regs, &data);

		status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;



		/*

arch/x86/events/intel/ds.c

+25 −22
Original line number Diff line number Diff line
@@ -670,7 +670,9 @@ unlock: 


static inline void intel_pmu_drain_pebs_buffer(void)

{

	x86_pmu.drain_pebs(NULL);

	struct perf_sample_data data;



	x86_pmu.drain_pebs(NULL, &data);

}



/*
@@ -1719,8 +1721,10 @@ intel_pmu_save_and_restart_reload(struct perf_event *event, int count) 
	return 0;

}



static void __intel_pmu_pebs_event(struct perf_event *event,

static __always_inline void

__intel_pmu_pebs_event(struct perf_event *event,

		       struct pt_regs *iregs,

		       struct perf_sample_data *data,

		       void *base, void *top,

		       int bit, int count,

		       void (*setup_sample)(struct perf_event *,
@@ -1731,7 +1735,6 @@ static void __intel_pmu_pebs_event(struct perf_event *event, 
{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct hw_perf_event *hwc = &event->hw;

	struct perf_sample_data data;

	struct x86_perf_regs perf_regs;

	struct pt_regs *regs = &perf_regs.regs;

	void *at = get_next_pebs_record_by_bit(base, top, bit);
@@ -1752,14 +1755,14 @@ static void __intel_pmu_pebs_event(struct perf_event *event, 
		iregs = &dummy_iregs;



	while (count > 1) {

		setup_sample(event, iregs, at, &data, regs);

		perf_event_output(event, &data, regs);

		setup_sample(event, iregs, at, data, regs);

		perf_event_output(event, data, regs);

		at += cpuc->pebs_record_size;

		at = get_next_pebs_record_by_bit(at, top, bit);

		count--;

	}



	setup_sample(event, iregs, at, &data, regs);

	setup_sample(event, iregs, at, data, regs);

	if (iregs == &dummy_iregs) {

		/*

		 * The PEBS records may be drained in the non-overflow context,
@@ -1767,18 +1770,18 @@ static void __intel_pmu_pebs_event(struct perf_event *event, 
		 * last record the same as other PEBS records, and doesn't

		 * invoke the generic overflow handler.

		 */

		perf_event_output(event, &data, regs);

		perf_event_output(event, data, regs);

	} else {

		/*

		 * All but the last records are processed.

		 * The last one is left to be able to call the overflow handler.

		 */

		if (perf_event_overflow(event, &data, regs))

		if (perf_event_overflow(event, data, regs))

			x86_pmu_stop(event, 0);

	}

}



static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)

static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct debug_store *ds = cpuc->ds;
@@ -1812,7 +1815,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs) 
		return;

	}



	__intel_pmu_pebs_event(event, iregs, at, top, 0, n,

	__intel_pmu_pebs_event(event, iregs, data, at, top, 0, n,

			       setup_pebs_fixed_sample_data);

}
@@ -1835,7 +1838,7 @@ static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, int 
	}

}



static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)

static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct debug_store *ds = cpuc->ds;
@@ -1942,14 +1945,14 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs) 
		}



		if (counts[bit]) {

			__intel_pmu_pebs_event(event, iregs, base,

			__intel_pmu_pebs_event(event, iregs, data, base,

					       top, bit, counts[bit],

					       setup_pebs_fixed_sample_data);

		}

	}

}



static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs)

static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)

{

	short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1997,7 +2000,7 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs) 
		if (WARN_ON_ONCE(!event->attr.precise_ip))

			continue;



		__intel_pmu_pebs_event(event, iregs, base,

		__intel_pmu_pebs_event(event, iregs, data, base,

				       top, bit, counts[bit],

				       setup_pebs_adaptive_sample_data);

	}

arch/x86/events/perf_event.h

+1 −1
Original line number Diff line number Diff line
@@ -727,7 +727,7 @@ struct x86_pmu { 
	int		pebs_record_size;

	int		pebs_buffer_size;

	int		max_pebs_events;

	void		(*drain_pebs)(struct pt_regs *regs);

	void		(*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);

	struct event_constraint *pebs_constraints;

	void		(*pebs_aliases)(struct perf_event *event);

	unsigned long	large_pebs_flags;

CRA Git | Maintained and supported by SUSTech CRA and CCSE