Commit e7c15cd8 authored by Steven Rostedt (Red Hat)'s avatar Steven Rostedt (Red Hat) Committed by Steven Rostedt
Browse files

tracing: Added hardware latency tracer 



The hardware latency tracer has been in the PREEMPT_RT patch for some time.
It is used to detect possible SMIs or any other hardware interruptions that
the kernel is unaware of. Note, NMIs may also be detected, but that may be
good to note as well.

The logic is pretty simple. It simply creates a thread that spins on a
single CPU for a specified amount of time (width) within a periodic window
(window). These numbers may be adjusted by their cooresponding names in

   /sys/kernel/tracing/hwlat_detector/

The defaults are window = 1000000 us (1 second)
                 width  =  500000 us (1/2 second)

The loop consists of:

	t1 = trace_clock_local();
	t2 = trace_clock_local();

Where trace_clock_local() is a variant of sched_clock().

The difference of t2 - t1 is recorded as the "inner" timestamp and also the
timestamp  t1 - prev_t2 is recorded as the "outer" timestamp. If either of
these differences are greater than the time denoted in
/sys/kernel/tracing/tracing_thresh then it records the event.

When this tracer is started, and tracing_thresh is zero, it changes to the
default threshold of 10 us.

The hwlat tracer in the PREEMPT_RT patch was originally written by
Jon Masters. I have modified it quite a bit and turned it into a
tracer.

Based-on-code-by: default avatarJon Masters <jcm@redhat.com>
Signed-off-by: default avatarSteven Rostedt <rostedt@goodmis.org>
parent 8861dd30

kernel/trace/Kconfig

+35 −0
Original line number Diff line number Diff line
@@ -221,6 +221,41 @@ config SCHED_TRACER 
	  This tracer tracks the latency of the highest priority task

	  to be scheduled in, starting from the point it has woken up.



config HWLAT_TRACER

	bool "Tracer to detect hardware latencies (like SMIs)"

	select GENERIC_TRACER

	help

	 This tracer, when enabled will create one or more kernel threads,

	 depening on what the cpumask file is set to, which each thread

	 spinning in a loop looking for interruptions caused by

	 something other than the kernel. For example, if a

	 System Management Interrupt (SMI) takes a noticeable amount of

	 time, this tracer will detect it. This is useful for testing

	 if a system is reliable for Real Time tasks.



	 Some files are created in the tracing directory when this

	 is enabled:



	   hwlat_detector/width   - time in usecs for how long to spin for

	   hwlat_detector/window  - time in usecs between the start of each

				     iteration



	 A kernel thread is created that will spin with interrupts disabled

	 for "width" microseconds in every "widow" cycle. It will not spin

	 for "window - width" microseconds, where the system can

	 continue to operate.



	 The output will appear in the trace and trace_pipe files.



	 When the tracer is not running, it has no affect on the system,

	 but when it is running, it can cause the system to be

	 periodically non responsive. Do not run this tracer on a

	 production system.



	 To enable this tracer, echo in "hwlat" into the current_tracer

	 file. Every time a latency is greater than tracing_thresh, it will

	 be recorded into the ring buffer.



config ENABLE_DEFAULT_TRACERS

	bool "Trace process context switches and events"

	depends on !GENERIC_TRACER

kernel/trace/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o 
obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o

obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o

obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o

obj-$(CONFIG_HWLAT_TRACER) += trace_hwlat.o

obj-$(CONFIG_NOP_TRACER) += trace_nop.o

obj-$(CONFIG_STACK_TRACER) += trace_stack.o

obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o

kernel/trace/trace.c

+1 −1
Original line number Diff line number Diff line
@@ -1047,7 +1047,7 @@ void disable_trace_on_warning(void) 
 *

 * Shows real state of the ring buffer if it is enabled or not.

 */

static int tracer_tracing_is_on(struct trace_array *tr)

int tracer_tracing_is_on(struct trace_array *tr)

{

	if (tr->trace_buffer.buffer)

		return ring_buffer_record_is_on(tr->trace_buffer.buffer);

kernel/trace/trace.h

+3 −0
Original line number Diff line number Diff line
@@ -38,6 +38,7 @@ enum trace_type { 
	TRACE_USER_STACK,

	TRACE_BLK,

	TRACE_BPUTS,

	TRACE_HWLAT,



	__TRACE_LAST_TYPE,

};
@@ -326,6 +327,7 @@ extern void __ftrace_bad_type(void); 
		IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT);	\

		IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT);	\

		IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS);	\

		IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT);	\

		IF_ASSIGN(var, ent, struct trace_mmiotrace_rw,		\

			  TRACE_MMIO_RW);				\

		IF_ASSIGN(var, ent, struct trace_mmiotrace_map,		\
@@ -571,6 +573,7 @@ void tracing_reset_current(int cpu); 
void tracing_reset_all_online_cpus(void);

int tracing_open_generic(struct inode *inode, struct file *filp);

bool tracing_is_disabled(void);

int tracer_tracing_is_on(struct trace_array *tr);

struct dentry *trace_create_file(const char *name,

				 umode_t mode,

				 struct dentry *parent,

kernel/trace/trace_entries.h

+23 −0
Original line number Diff line number Diff line
@@ -322,3 +322,26 @@ FTRACE_ENTRY(branch, trace_branch, 
	FILTER_OTHER

);





FTRACE_ENTRY(hwlat, hwlat_entry,



	TRACE_HWLAT,



	F_STRUCT(

		__field(	u64,			duration	)

		__field(	u64,			outer_duration	)

		__field_struct( struct timespec,	timestamp	)

		__field_desc(	long,	timestamp,	tv_sec		)

		__field_desc(	long,	timestamp,	tv_nsec		)

		__field(	unsigned int,		seqnum		)

	),



	F_printk("cnt:%u\tts:%010lu.%010lu\tinner:%llu\touter:%llu\n",

		 __entry->seqnum,

		 __entry->tv_sec,

		 __entry->tv_nsec,

		 __entry->duration,

		 __entry->outer_duration),



	FILTER_OTHER

);

CRA Git | Maintained and supported by SUSTech CRA and CCSE