perf/x86/intel/bts: Add BTS PMU driver (8062382c) · Commits · 戴 / test

arch/x86/kernel/cpu/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -40,7 +40,7 @@ endif
		obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_p6.o perf_event_knc.o perf_event_p4.o
		obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_lbr.o perf_event_intel_ds.o perf_event_intel.o
		obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_rapl.o perf_event_intel_cqm.o
		obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_pt.o
		obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_pt.o perf_event_intel_bts.o

		obj-$(CONFIG_PERF_EVENTS_INTEL_UNCORE) += perf_event_intel_uncore.o \
		perf_event_intel_uncore_snb.o \

arch/x86/kernel/cpu/perf_event.h

+7 −0

Original line number	Diff line number	Diff line
		@@ -410,6 +410,7 @@ union x86_pmu_config {

		enum {
		x86_lbr_exclusive_lbr,
		x86_lbr_exclusive_bts,
		x86_lbr_exclusive_pt,
		x86_lbr_exclusive_max,
		};
		@@ -810,6 +811,12 @@ int intel_pmu_setup_lbr_filter(struct perf_event *event);

		void intel_pt_interrupt(void);

		int intel_bts_interrupt(void);

		void intel_bts_enable_local(void);

		void intel_bts_disable_local(void);

		int p4_pmu_init(void);

		int p6_pmu_init(void);

arch/x86/kernel/cpu/perf_event_intel.c

+5 −1

Original line number	Diff line number	Diff line
		@@ -1242,6 +1242,8 @@ static void intel_pmu_disable_all(void)

		if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
		intel_pmu_disable_bts();
		else
		intel_bts_disable_local();

		intel_pmu_pebs_disable_all();
		intel_pmu_lbr_disable_all();
		@@ -1264,7 +1266,8 @@ static void intel_pmu_enable_all(int added)
		return;

		intel_pmu_enable_bts(event->hw.config);
		}
		} else
		intel_bts_enable_local();
		}

		/*
		@@ -1550,6 +1553,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
		apic_write(APIC_LVTPC, APIC_DM_NMI);
		intel_pmu_disable_all();
		handled = intel_pmu_drain_bts_buffer();
		handled += intel_bts_interrupt();
		status = intel_pmu_get_status();
		if (!status)
		goto done;

arch/x86/kernel/cpu/perf_event_intel_bts.c

0 → 100644

+525 −0

Original line number	Diff line number	Diff line
		/*
		* BTS PMU driver for perf
		* Copyright (c) 2013-2014, Intel Corporation.
		*
		* This program is free software; you can redistribute it and/or modify it
		* under the terms and conditions of the GNU General Public License,
		* version 2, as published by the Free Software Foundation.
		*
		* This program is distributed in the hope it will be useful, but WITHOUT
		* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
		* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
		* more details.
		*/

		#undef DEBUG

		#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

		#include <linux/bitops.h>
		#include <linux/types.h>
		#include <linux/slab.h>
		#include <linux/debugfs.h>
		#include <linux/device.h>
		#include <linux/coredump.h>

		#include <asm-generic/sizes.h>
		#include <asm/perf_event.h>

		#include "perf_event.h"

		struct bts_ctx {
		struct perf_output_handle handle;
		struct debug_store ds_back;
		int started;
		};

		static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);

		#define BTS_RECORD_SIZE 24
		#define BTS_SAFETY_MARGIN 4080

		struct bts_phys {
		struct page *page;
		unsigned long size;
		unsigned long offset;
		unsigned long displacement;
		};

		struct bts_buffer {
		size_t real_size; /* multiple of BTS_RECORD_SIZE */
		unsigned int nr_pages;
		unsigned int nr_bufs;
		unsigned int cur_buf;
		bool snapshot;
		local_t data_size;
		local_t lost;
		local_t head;
		unsigned long end;
		void **data_pages;
		struct bts_phys buf[0];
		};

		struct pmu bts_pmu;

		void intel_pmu_enable_bts(u64 config);
		void intel_pmu_disable_bts(void);

		static size_t buf_size(struct page *page)
		{
		return 1 << (PAGE_SHIFT + page_private(page));
		}

		static void *
		bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
		{
		struct bts_buffer *buf;
		struct page *page;
		int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
		unsigned long offset;
		size_t size = nr_pages << PAGE_SHIFT;
		int pg, nbuf, pad;

		/* count all the high order buffers */
		for (pg = 0, nbuf = 0; pg < nr_pages;) {
		page = virt_to_page(pages[pg]);
		if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
		return NULL;
		pg += 1 << page_private(page);
		nbuf++;
		}

		/*
		* to avoid interrupts in overwrite mode, only allow one physical
		*/
		if (overwrite && nbuf > 1)
		return NULL;

		buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
		if (!buf)
		return NULL;

		buf->nr_pages = nr_pages;
		buf->nr_bufs = nbuf;
		buf->snapshot = overwrite;
		buf->data_pages = pages;
		buf->real_size = size - size % BTS_RECORD_SIZE;

		for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
		unsigned int __nr_pages;

		page = virt_to_page(pages[pg]);
		__nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
		buf->buf[nbuf].page = page;
		buf->buf[nbuf].offset = offset;
		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
		buf->buf[nbuf].size -= pad;

		pg += __nr_pages;
		offset += __nr_pages << PAGE_SHIFT;
		}

		return buf;
		}

		static void bts_buffer_free_aux(void *data)
		{
		kfree(data);
		}

		static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
		{
		return buf->buf[idx].offset + buf->buf[idx].displacement;
		}

		static void
		bts_config_buffer(struct bts_buffer *buf)
		{
		int cpu = raw_smp_processor_id();
		struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
		struct bts_phys *phys = &buf->buf[buf->cur_buf];
		unsigned long index, thresh = 0, end = phys->size;
		struct page *page = phys->page;

		index = local_read(&buf->head);

		if (!buf->snapshot) {
		if (buf->end < phys->offset + buf_size(page))
		end = buf->end - phys->offset - phys->displacement;

		index -= phys->offset + phys->displacement;

		if (end - index > BTS_SAFETY_MARGIN)
		thresh = end - BTS_SAFETY_MARGIN;
		else if (end - index > BTS_RECORD_SIZE)
		thresh = end - BTS_RECORD_SIZE;
		else
		thresh = end;
		}

		ds->bts_buffer_base = (u64)page_address(page) + phys->displacement;
		ds->bts_index = ds->bts_buffer_base + index;
		ds->bts_absolute_maximum = ds->bts_buffer_base + end;
		ds->bts_interrupt_threshold = !buf->snapshot
		? ds->bts_buffer_base + thresh
		: ds->bts_absolute_maximum + BTS_RECORD_SIZE;
		}

		static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
		{
		unsigned long index = head - phys->offset;

		memset(page_address(phys->page) + index, 0, phys->size - index);
		}

		static bool bts_buffer_is_full(struct bts_buffer buf, struct bts_ctx bts)
		{
		if (buf->snapshot)
		return false;

		if (local_read(&buf->data_size) >= bts->handle.size \|\|
		bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)
		return true;

		return false;
		}

		static void bts_update(struct bts_ctx *bts)
		{
		int cpu = raw_smp_processor_id();
		struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
		struct bts_buffer *buf = perf_get_aux(&bts->handle);
		unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;

		if (!buf)
		return;

		head = index + bts_buffer_offset(buf, buf->cur_buf);
		old = local_xchg(&buf->head, head);

		if (!buf->snapshot) {
		if (old == head)
		return;

		if (ds->bts_index >= ds->bts_absolute_maximum)
		local_inc(&buf->lost);

		/*
		* old and head are always in the same physical buffer, so we
		* can subtract them to get the data size.
		*/
		local_add(head - old, &buf->data_size);
		} else {
		local_set(&buf->data_size, head);
		}
		}

		static void __bts_event_start(struct perf_event *event)
		{
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
		struct bts_buffer *buf = perf_get_aux(&bts->handle);
		u64 config = 0;

		if (!buf \|\| bts_buffer_is_full(buf, bts))
		return;

		event->hw.state = 0;

		if (!buf->snapshot)
		config \|= ARCH_PERFMON_EVENTSEL_INT;
		if (!event->attr.exclude_kernel)
		config \|= ARCH_PERFMON_EVENTSEL_OS;
		if (!event->attr.exclude_user)
		config \|= ARCH_PERFMON_EVENTSEL_USR;

		bts_config_buffer(buf);

		/*
		* local barrier to make sure that ds configuration made it
		* before we enable BTS
		*/
		wmb();

		intel_pmu_enable_bts(config);
		}

		static void bts_event_start(struct perf_event *event, int flags)
		{
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);

		__bts_event_start(event);

		/* PMI handler: this counter is running and likely generating PMIs */
		ACCESS_ONCE(bts->started) = 1;
		}

		static void __bts_event_stop(struct perf_event *event)
		{
		/*
		* No extra synchronization is mandated by the documentation to have
		* BTS data stores globally visible.
		*/
		intel_pmu_disable_bts();

		if (event->hw.state & PERF_HES_STOPPED)
		return;

		ACCESS_ONCE(event->hw.state) \|= PERF_HES_STOPPED;
		}

		static void bts_event_stop(struct perf_event *event, int flags)
		{
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);

		/* PMI handler: don't restart this counter */
		ACCESS_ONCE(bts->started) = 0;

		__bts_event_stop(event);

		if (flags & PERF_EF_UPDATE)
		bts_update(bts);
		}

		void intel_bts_enable_local(void)
		{
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);

		if (bts->handle.event && bts->started)
		__bts_event_start(bts->handle.event);
		}

		void intel_bts_disable_local(void)
		{
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);

		if (bts->handle.event)
		__bts_event_stop(bts->handle.event);
		}

		static int
		bts_buffer_reset(struct bts_buffer buf, struct perf_output_handle handle)
		{
		unsigned long head, space, next_space, pad, gap, skip, wakeup;
		unsigned int next_buf;
		struct bts_phys phys, next_phys;
		int ret;

		if (buf->snapshot)
		return 0;

		head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
		if (WARN_ON_ONCE(head != local_read(&buf->head)))
		return -EINVAL;

		phys = &buf->buf[buf->cur_buf];
		space = phys->offset + phys->displacement + phys->size - head;
		pad = space;
		if (space > handle->size) {
		space = handle->size;
		space -= space % BTS_RECORD_SIZE;
		}
		if (space <= BTS_SAFETY_MARGIN) {
		/* See if next phys buffer has more space */
		next_buf = buf->cur_buf + 1;
		if (next_buf >= buf->nr_bufs)
		next_buf = 0;
		next_phys = &buf->buf[next_buf];
		gap = buf_size(phys->page) - phys->displacement - phys->size +
		next_phys->displacement;
		skip = pad + gap;
		if (handle->size >= skip) {
		next_space = next_phys->size;
		if (next_space + skip > handle->size) {
		next_space = handle->size - skip;
		next_space -= next_space % BTS_RECORD_SIZE;
		}
		if (next_space > space \|\| !space) {
		if (pad)
		bts_buffer_pad_out(phys, head);
		ret = perf_aux_output_skip(handle, skip);
		if (ret)
		return ret;
		/* Advance to next phys buffer */
		phys = next_phys;
		space = next_space;
		head = phys->offset + phys->displacement;
		/*
		* After this, cur_buf and head won't match ds
		* anymore, so we must not be racing with
		* bts_update().
		*/
		buf->cur_buf = next_buf;
		local_set(&buf->head, head);
		}
		}
		}

		/* Don't go far beyond wakeup watermark */
		wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
		handle->head;
		if (space > wakeup) {
		space = wakeup;
		space -= space % BTS_RECORD_SIZE;
		}

		buf->end = head + space;

		/*
		* If we have no space, the lost notification would have been sent when
		* we hit absolute_maximum - see bts_update()
		*/
		if (!space)
		return -ENOSPC;

		return 0;
		}

		int intel_bts_interrupt(void)
		{
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
		struct perf_event *event = bts->handle.event;
		struct bts_buffer *buf;
		s64 old_head;
		int err;

		if (!event \|\| !bts->started)
		return 0;

		buf = perf_get_aux(&bts->handle);
		/*
		* Skip snapshot counters: they don't use the interrupt, but
		* there's no other way of telling, because the pointer will
		* keep moving
		*/
		if (!buf \|\| buf->snapshot)
		return 0;

		old_head = local_read(&buf->head);
		bts_update(bts);

		/* no new data */
		if (old_head == local_read(&buf->head))
		return 0;

		perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
		!!local_xchg(&buf->lost, 0));

		buf = perf_aux_output_begin(&bts->handle, event);
		if (!buf)
		return 1;

		err = bts_buffer_reset(buf, &bts->handle);
		if (err)
		perf_aux_output_end(&bts->handle, 0, false);

		return 1;
		}

		static void bts_event_del(struct perf_event *event, int mode)
		{
		struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
		struct bts_buffer *buf = perf_get_aux(&bts->handle);

		bts_event_stop(event, PERF_EF_UPDATE);

		if (buf) {
		if (buf->snapshot)
		bts->handle.head =
		local_xchg(&buf->data_size,
		buf->nr_pages << PAGE_SHIFT);
		perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
		!!local_xchg(&buf->lost, 0));
		}

		cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
		cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
		cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
		cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
		}

		static int bts_event_add(struct perf_event *event, int mode)
		{
		struct bts_buffer *buf;
		struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
		struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
		struct hw_perf_event *hwc = &event->hw;
		int ret = -EBUSY;

		event->hw.state = PERF_HES_STOPPED;

		if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
		return -EBUSY;

		if (bts->handle.event)
		return -EBUSY;

		buf = perf_aux_output_begin(&bts->handle, event);
		if (!buf)
		return -EINVAL;

		ret = bts_buffer_reset(buf, &bts->handle);
		if (ret) {
		perf_aux_output_end(&bts->handle, 0, false);
		return ret;
		}

		bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
		bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
		bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;

		if (mode & PERF_EF_START) {
		bts_event_start(event, 0);
		if (hwc->state & PERF_HES_STOPPED) {
		bts_event_del(event, 0);
		return -EBUSY;
		}
		}

		return 0;
		}

		static void bts_event_destroy(struct perf_event *event)
		{
		x86_del_exclusive(x86_lbr_exclusive_bts);
		}

		static int bts_event_init(struct perf_event *event)
		{
		if (event->attr.type != bts_pmu.type)
		return -ENOENT;

		if (x86_add_exclusive(x86_lbr_exclusive_bts))
		return -EBUSY;

		event->destroy = bts_event_destroy;

		return 0;
		}

		static void bts_event_read(struct perf_event *event)
		{
		}

		static __init int bts_init(void)
		{
		if (!boot_cpu_has(X86_FEATURE_DTES64) \|\| !x86_pmu.bts)
		return -ENODEV;

		bts_pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG \| PERF_PMU_CAP_ITRACE;
		bts_pmu.task_ctx_nr = perf_sw_context;
		bts_pmu.event_init = bts_event_init;
		bts_pmu.add = bts_event_add;
		bts_pmu.del = bts_event_del;
		bts_pmu.start = bts_event_start;
		bts_pmu.stop = bts_event_stop;
		bts_pmu.read = bts_event_read;
		bts_pmu.setup_aux = bts_buffer_setup_aux;
		bts_pmu.free_aux = bts_buffer_free_aux;

		return perf_pmu_register(&bts_pmu, "intel_bts", -1);
		}

		module_init(bts_init);

arch/x86/kernel/cpu/perf_event_intel_ds.c

+2 −1

Original line number	Diff line number	Diff line
		@@ -461,6 +461,7 @@ void intel_pmu_enable_bts(u64 config)

		debugctlmsr \|= DEBUGCTLMSR_TR;
		debugctlmsr \|= DEBUGCTLMSR_BTS;
		if (config & ARCH_PERFMON_EVENTSEL_INT)
		debugctlmsr \|= DEBUGCTLMSR_BTINT;

		if (!(config & ARCH_PERFMON_EVENTSEL_OS))

Admin message