Commit 80a4ae20 authored by Benjamin Herrenschmidt's avatar Benjamin Herrenschmidt Committed by Michael Ellerman
Browse files

powerpc/mm: Workaround Nest MMU bug with TLB invalidations 



On POWER9 the Nest MMU may fail to invalidate some translations when
doing a tlbie "by PID" or "by LPID" that is targeted at the TLB only
and not the page walk cache.

This works around it by forcing such invalidations to escalate to
RIC=2 (full invalidation of TLB *and* PWC) when a coprocessor is in
use for the context.

Fixes: 03b8abed ("cxl: Enable global TLBIs for cxl contexts")
Cc: stable@vger.kernel.org # v4.15+
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: default avatarBalbir Singh <bsingharora@gmail.com>
[balbirs: fixed spelling and coding style to quiesce checkpatch.pl]
Tested-by: default avatarBalbir Singh <bsingharora@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent aff6f8cb

arch/powerpc/mm/tlb-radix.c

+43 −7
Original line number Diff line number Diff line
@@ -151,7 +151,23 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) 
static inline void _tlbie_pid(unsigned long pid, unsigned long ric)

{

	asm volatile("ptesync": : :"memory");

	__tlbie_pid(pid, ric);



	/*

	 * Workaround the fact that the "ric" argument to __tlbie_pid

	 * must be a compile-time contraint to match the "i" constraint

	 * in the asm statement.

	 */

	switch (ric) {

	case RIC_FLUSH_TLB:

		__tlbie_pid(pid, RIC_FLUSH_TLB);

		break;

	case RIC_FLUSH_PWC:

		__tlbie_pid(pid, RIC_FLUSH_PWC);

		break;

	case RIC_FLUSH_ALL:

	default:

		__tlbie_pid(pid, RIC_FLUSH_ALL);

	}

	asm volatile("eieio; tlbsync; ptesync": : :"memory");

}
@@ -311,6 +327,16 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd 
}

EXPORT_SYMBOL(radix__local_flush_tlb_page);



static bool mm_needs_flush_escalation(struct mm_struct *mm)

{

	/*

	 * P9 nest MMU has issues with the page walk cache

	 * caching PTEs and not flushing them properly when

	 * RIC = 0 for a PID/LPID invalidate

	 */

	return atomic_read(&mm->context.copros) != 0;

}



#ifdef CONFIG_SMP

void radix__flush_tlb_mm(struct mm_struct *mm)

{
@@ -321,9 +347,12 @@ void radix__flush_tlb_mm(struct mm_struct *mm) 
		return;



	preempt_disable();

	if (!mm_is_thread_local(mm))

		_tlbie_pid(pid, RIC_FLUSH_TLB);

	if (!mm_is_thread_local(mm)) {

		if (mm_needs_flush_escalation(mm))

			_tlbie_pid(pid, RIC_FLUSH_ALL);

		else

			_tlbie_pid(pid, RIC_FLUSH_TLB);

	} else

		_tlbiel_pid(pid, RIC_FLUSH_TLB);

	preempt_enable();

}
@@ -435,10 +464,14 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 
	}



	if (full) {

		if (local)

		if (local) {

			_tlbiel_pid(pid, RIC_FLUSH_TLB);

		} else {

			if (mm_needs_flush_escalation(mm))

				_tlbie_pid(pid, RIC_FLUSH_ALL);

			else

				_tlbie_pid(pid, RIC_FLUSH_TLB);

		}

	} else {

		bool hflush = false;

		unsigned long hstart, hend;
@@ -548,6 +581,9 @@ static inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, 
	}



	if (full) {

		if (!local && mm_needs_flush_escalation(mm))

			also_pwc = true;



		if (local)

			_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);

		else

CRA Git | Maintained and supported by SUSTech CRA and CCSE