Commit 38fef73c authored by Max Filippov's avatar Max Filippov
Browse files

xtensa: implement fake NMI 



In case perf IRQ is the highest of the medium-level IRQs, and is alone
on its level, it may be treated as NMI:
- LOCKLEVEL is defined to be one level less than EXCM level,
- IRQ masking never lowers current IRQ level,
- new fake exception cause code, EXCCAUSE_MAPPED_NMI is assigned to that
  IRQ; new second level exception handler, do_nmi, assigned to it
  handles it as NMI,
- atomic operations in configurations without s32c1i still need to mask
  all interrupts.

Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarMax Filippov <jcmvbkbc@gmail.com>
parent 98e29832

arch/xtensa/include/asm/atomic.h

+5 −5
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ 
 *

 * Locking interrupts looks like this:

 *

 *    rsil a15, LOCKLEVEL

 *    rsil a15, TOPLEVEL

 *    <code>

 *    wsr  a15, PS

 *    rsync
@@ -106,7 +106,7 @@ static inline void atomic_##op(int i, atomic_t * v) \ 
	unsigned int vval;						\

									\

	__asm__ __volatile__(						\

			"       rsil    a15, "__stringify(LOCKLEVEL)"\n"\

			"       rsil    a15, "__stringify(TOPLEVEL)"\n"\

			"       l32i    %0, %2, 0\n"			\

			"       " #op " %0, %0, %1\n"			\

			"       s32i    %0, %2, 0\n"			\
@@ -124,7 +124,7 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ 
	unsigned int vval;						\

									\

	__asm__ __volatile__(						\

			"       rsil    a15,"__stringify(LOCKLEVEL)"\n"	\

			"       rsil    a15,"__stringify(TOPLEVEL)"\n"	\

			"       l32i    %0, %2, 0\n"			\

			"       " #op " %0, %0, %1\n"			\

			"       s32i    %0, %2, 0\n"			\
@@ -272,7 +272,7 @@ static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) 
	unsigned int vval;



	__asm__ __volatile__(

			"       rsil    a15,"__stringify(LOCKLEVEL)"\n"

			"       rsil    a15,"__stringify(TOPLEVEL)"\n"

			"       l32i    %0, %2, 0\n"

			"       xor     %1, %4, %3\n"

			"       and     %0, %0, %4\n"
@@ -306,7 +306,7 @@ static inline void atomic_set_mask(unsigned int mask, atomic_t *v) 
	unsigned int vval;



	__asm__ __volatile__(

			"       rsil    a15,"__stringify(LOCKLEVEL)"\n"

			"       rsil    a15,"__stringify(TOPLEVEL)"\n"

			"       l32i    %0, %2, 0\n"

			"       or      %0, %0, %1\n"

			"       s32i    %0, %2, 0\n"

arch/xtensa/include/asm/cmpxchg.h

+2 −2
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ __cmpxchg_u32(volatile int *p, int old, int new) 
	return new;

#else

	__asm__ __volatile__(

			"       rsil    a15, "__stringify(LOCKLEVEL)"\n"

			"       rsil    a15, "__stringify(TOPLEVEL)"\n"

			"       l32i    %0, %1, 0\n"

			"       bne     %0, %2, 1f\n"

			"       s32i    %3, %1, 0\n"
@@ -123,7 +123,7 @@ static inline unsigned long xchg_u32(volatile int * m, unsigned long val) 
#else

	unsigned long tmp;

	__asm__ __volatile__(

			"       rsil    a15, "__stringify(LOCKLEVEL)"\n"

			"       rsil    a15, "__stringify(TOPLEVEL)"\n"

			"       l32i    %0, %1, 0\n"

			"       s32i    %2, %1, 0\n"

			"       wsr     a15, ps\n"

arch/xtensa/include/asm/irqflags.h

+21 −1
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
 * for more details.

 *

 * Copyright (C) 2001 - 2005 Tensilica Inc.

 * Copyright (C) 2015 Cadence Design Systems Inc.

 */



#ifndef _XTENSA_IRQFLAGS_H
@@ -23,8 +24,27 @@ static inline unsigned long arch_local_save_flags(void) 
static inline unsigned long arch_local_irq_save(void)

{

	unsigned long flags;

#if XTENSA_FAKE_NMI

#if defined(CONFIG_DEBUG_KERNEL) && (LOCKLEVEL | TOPLEVEL) >= XCHAL_DEBUGLEVEL

	unsigned long tmp;



	asm volatile("rsr	%0, ps\t\n"

		     "extui	%1, %0, 0, 4\t\n"

		     "bgei	%1, "__stringify(LOCKLEVEL)", 1f\t\n"

		     "rsil	%0, "__stringify(LOCKLEVEL)"\n"

		     "1:"

		     : "=a" (flags), "=a" (tmp) :: "memory");

#else

	asm volatile("rsr	%0, ps\t\n"

		     "or	%0, %0, %1\t\n"

		     "xsr	%0, ps\t\n"

		     "rsync"

		     : "=&a" (flags) : "a" (LOCKLEVEL) : "memory");

#endif

#else

	asm volatile("rsil	%0, "__stringify(LOCKLEVEL)

		     : "=a" (flags) :: "memory");

#endif

	return flags;

}

arch/xtensa/include/asm/processor.h

+29 −2
Original line number Diff line number Diff line 
/*

 * include/asm-xtensa/processor.h

 *

 * This file is subject to the terms and conditions of the GNU General Public

 * License.  See the file "COPYING" in the main directory of this archive

 * for more details.

 *

 * Copyright (C) 2001 - 2008 Tensilica Inc.

 * Copyright (C) 2015 Cadence Design Systems Inc.

 */



#ifndef _XTENSA_PROCESSOR_H
@@ -44,6 +43,14 @@ 
#define STACK_TOP	TASK_SIZE

#define STACK_TOP_MAX	STACK_TOP



/*

 * General exception cause assigned to fake NMI. Fake NMI needs to be handled

 * differently from other interrupts, but it uses common kernel entry/exit

 * code.

 */



#define EXCCAUSE_MAPPED_NMI	62



/*

 * General exception cause assigned to debug exceptions. Debug exceptions go

 * to their own vector, rather than the general exception vectors (user,
@@ -65,10 +72,30 @@ 


#define VALID_DOUBLE_EXCEPTION_ADDRESS	64



#define XTENSA_INT_LEVEL(intno) _XTENSA_INT_LEVEL(intno)

#define _XTENSA_INT_LEVEL(intno) XCHAL_INT##intno##_LEVEL



#define XTENSA_INTLEVEL_MASK(level) _XTENSA_INTLEVEL_MASK(level)

#define _XTENSA_INTLEVEL_MASK(level) (XCHAL_INTLEVEL##level##_MASK)



#define IS_POW2(v) (((v) & ((v) - 1)) == 0)



#define PROFILING_INTLEVEL XTENSA_INT_LEVEL(XCHAL_PROFILING_INTERRUPT)



/* LOCKLEVEL defines the interrupt level that masks all

 * general-purpose interrupts.

 */

#if defined(CONFIG_XTENSA_VARIANT_HAVE_PERF_EVENTS) && \

	defined(XCHAL_PROFILING_INTERRUPT) && \

	PROFILING_INTLEVEL == XCHAL_EXCM_LEVEL && \

	XCHAL_EXCM_LEVEL > 1 && \

	IS_POW2(XTENSA_INTLEVEL_MASK(PROFILING_INTLEVEL))

#define LOCKLEVEL (XCHAL_EXCM_LEVEL - 1)

#else

#define LOCKLEVEL XCHAL_EXCM_LEVEL

#endif

#define TOPLEVEL XCHAL_EXCM_LEVEL

#define XTENSA_FAKE_NMI (LOCKLEVEL < TOPLEVEL)



/* WSBITS and WBBITS are the width of the WINDOWSTART and WINDOWBASE

 * registers

arch/xtensa/kernel/entry.S

+79 −14
Original line number Diff line number Diff line 
/*

 * arch/xtensa/kernel/entry.S

 *

 * Low-level exception handling

 *

 * This file is subject to the terms and conditions of the GNU General Public
@@ -8,6 +6,7 @@ 
 * for more details.

 *

 * Copyright (C) 2004 - 2008 by Tensilica Inc.

 * Copyright (C) 2015 Cadence Design Systems Inc.

 *

 * Chris Zankel <chris@zankel.net>

 *
@@ -75,6 +74,27 @@ 
#endif

	.endm





	.macro	irq_save flags tmp

#if XTENSA_FAKE_NMI

#if defined(CONFIG_DEBUG_KERNEL) && (LOCKLEVEL | TOPLEVEL) >= XCHAL_DEBUGLEVEL

	rsr	\flags, ps

	extui	\tmp, \flags, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH

	bgei	\tmp, LOCKLEVEL, 99f

	rsil	\tmp, LOCKLEVEL

99:

#else

	movi	\tmp, LOCKLEVEL

	rsr	\flags, ps

	or	\flags, \flags, \tmp

	xsr	\flags, ps

	rsync

#endif

#else

	rsil	\flags, LOCKLEVEL

#endif

	.endm



/* ----------------- DEFAULT FIRST LEVEL EXCEPTION HANDLERS ----------------- */



/*
@@ -352,11 +372,11 @@ common_exception: 


	/* It is now save to restore the EXC_TABLE_FIXUP variable. */



	rsr	a0, exccause

	rsr	a2, exccause

	movi	a3, 0

	rsr	a2, excsave1

	s32i	a0, a1, PT_EXCCAUSE

	s32i	a3, a2, EXC_TABLE_FIXUP

	rsr	a0, excsave1

	s32i	a2, a1, PT_EXCCAUSE

	s32i	a3, a0, EXC_TABLE_FIXUP



	/* All unrecoverable states are saved on stack, now, and a1 is valid.

	 * Now we can allow exceptions again. In case we've got an interrupt
@@ -367,19 +387,46 @@ common_exception: 
	 */



	rsr	a3, ps

	addi	a0, a0, -EXCCAUSE_LEVEL1_INTERRUPT

	movi	a2, LOCKLEVEL

	s32i	a3, a1, PT_PS		# save ps



#if XTENSA_FAKE_NMI

	/* Correct PS needs to be saved in the PT_PS:

	 * - in case of exception or level-1 interrupt it's in the PS,

	 *   and is already saved.

	 * - in case of medium level interrupt it's in the excsave2.

	 */

	movi	a0, EXCCAUSE_MAPPED_NMI

	extui	a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH

	beq	a2, a0, .Lmedium_level_irq

	bnei	a2, EXCCAUSE_LEVEL1_INTERRUPT, .Lexception

	beqz	a3, .Llevel1_irq	# level-1 IRQ sets ps.intlevel to 0



.Lmedium_level_irq:

	rsr	a0, excsave2

	s32i	a0, a1, PT_PS		# save medium-level interrupt ps

	bgei	a3, LOCKLEVEL, .Lexception



.Llevel1_irq:

	movi	a3, LOCKLEVEL



.Lexception:

	movi	a0, 1 << PS_WOE_BIT

	or	a3, a3, a0

#else

	addi	a2, a2, -EXCCAUSE_LEVEL1_INTERRUPT

	movi	a0, LOCKLEVEL

	extui	a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH

					# a3 = PS.INTLEVEL

	moveqz	a3, a2, a0		# a3 = LOCKLEVEL iff interrupt

	moveqz	a3, a0, a2		# a3 = LOCKLEVEL iff interrupt

	movi	a2, 1 << PS_WOE_BIT

	or	a3, a3, a2

	rsr	a2, exccause

#endif



	/* restore return address (or 0 if return to userspace) */

	rsr	a0, depc

	xsr	a3, ps



	s32i	a3, a1, PT_PS		# save ps

	wsr	a3, ps

	rsync				# PS.WOE => rsync => overflow



	/* Save lbeg, lend */
@@ -417,8 +464,13 @@ common_exception: 
	.global common_exception_return

common_exception_return:



#if XTENSA_FAKE_NMI

	l32i	a2, a1, PT_EXCCAUSE

	movi	a3, EXCCAUSE_MAPPED_NMI

	beq	a2, a3, .LNMIexit

#endif

1:

	rsil	a2, LOCKLEVEL

	irq_save a2, a3

#ifdef CONFIG_TRACE_IRQFLAGS

	movi	a4, trace_hardirqs_off

	callx4	a4
@@ -481,6 +533,12 @@ common_exception_return: 
	j	1b

#endif



#if XTENSA_FAKE_NMI

.LNMIexit:

	l32i	a3, a1, PT_PS

	_bbci.l	a3, PS_UM_BIT, 4f

#endif



5:

#ifdef CONFIG_DEBUG_TLB_SANITY

	l32i	a4, a1, PT_DEPC
@@ -1564,6 +1622,13 @@ ENTRY(fast_second_level_miss) 
	rfde



9:	l32i	a0, a1, TASK_ACTIVE_MM	# unlikely case mm == 0

	bnez	a0, 8b



	/* Even more unlikely case active_mm == 0.

	 * We can get here with NMI in the middle of context_switch that

	 * touches vmalloc area.

	 */

	movi	a0, init_mm

	j	8b



#if (DCACHE_WAY_SIZE > PAGE_SIZE)
@@ -1867,7 +1932,7 @@ ENTRY(_switch_to) 


	/* Disable ints while we manipulate the stack pointer. */



	rsil	a14, LOCKLEVEL

	irq_save a14, a3

	rsync



	/* Switch CPENABLE */

CRA Git | Maintained and supported by SUSTech CRA and CCSE