Commit 8fed6200 authored by Dave Hansen's avatar Dave Hansen Committed by Peter Zijlstra
Browse files

x86/mm: Break out kernel address space handling 



The page fault handler (__do_page_fault())  basically has two sections:
one for handling faults in the kernel portion of the address space
and another for faults in the user portion of the address space.

But, these two parts don't stick out that well.  Let's make that more
clear from code separation and naming.  Pull kernel fault
handling into its own helper, and reflect that naming by renaming
spurious_fault() -> spurious_kernel_fault().

Also, rewrite the vmalloc() handling comment a bit.  It was a bit
stale and also glossed over the reserved bit handling.

Cc: x86@kernel.org
Cc: Jann Horn <jannh@google.com>
Cc: Sean Christopherson <sean.j.christopherson@intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Andy Lutomirski <luto@kernel.org>
Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: http://lkml.kernel.org/r/20180928160222.401F4E10@viggo.jf.intel.com
parent 164477c2

arch/x86/mm/fault.c

+62 −39
Original line number Diff line number Diff line
@@ -1032,7 +1032,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, 
	}

}



static int spurious_fault_check(unsigned long error_code, pte_t *pte)

static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte)

{

	if ((error_code & X86_PF_WRITE) && !pte_write(*pte))

		return 0;
@@ -1071,7 +1071,7 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte) 
 * (Optional Invalidation).

 */

static noinline int

spurious_fault(unsigned long error_code, unsigned long address)

spurious_kernel_fault(unsigned long error_code, unsigned long address)

{

	pgd_t *pgd;

	p4d_t *p4d;
@@ -1102,27 +1102,27 @@ spurious_fault(unsigned long error_code, unsigned long address) 
		return 0;



	if (p4d_large(*p4d))

		return spurious_fault_check(error_code, (pte_t *) p4d);

		return spurious_kernel_fault_check(error_code, (pte_t *) p4d);



	pud = pud_offset(p4d, address);

	if (!pud_present(*pud))

		return 0;



	if (pud_large(*pud))

		return spurious_fault_check(error_code, (pte_t *) pud);

		return spurious_kernel_fault_check(error_code, (pte_t *) pud);



	pmd = pmd_offset(pud, address);

	if (!pmd_present(*pmd))

		return 0;



	if (pmd_large(*pmd))

		return spurious_fault_check(error_code, (pte_t *) pmd);

		return spurious_kernel_fault_check(error_code, (pte_t *) pmd);



	pte = pte_offset_kernel(pmd, address);

	if (!pte_present(*pte))

		return 0;



	ret = spurious_fault_check(error_code, pte);

	ret = spurious_kernel_fault_check(error_code, pte);

	if (!ret)

		return 0;
@@ -1130,12 +1130,12 @@ spurious_fault(unsigned long error_code, unsigned long address) 
	 * Make sure we have permissions in PMD.

	 * If not, then there's a bug in the page tables:

	 */

	ret = spurious_fault_check(error_code, (pte_t *) pmd);

	ret = spurious_kernel_fault_check(error_code, (pte_t *) pmd);

	WARN_ONCE(!ret, "PMD has incorrect permission bits\n");



	return ret;

}

NOKPROBE_SYMBOL(spurious_fault);

NOKPROBE_SYMBOL(spurious_kernel_fault);



int show_unhandled_signals = 1;
@@ -1203,32 +1203,16 @@ static inline bool smap_violation(int error_code, struct pt_regs *regs) 
}



/*

 * This routine handles page faults.  It determines the address,

 * and the problem, and then passes it off to one of the appropriate

 * routines.

 * Called for all faults where 'address' is part of the kernel address

 * space.  Might get called for faults that originate from *code* that

 * ran in userspace or the kernel.

 */

static noinline void

__do_page_fault(struct pt_regs *regs, unsigned long hw_error_code,

static void

do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code,

		   unsigned long address)

{

	unsigned long sw_error_code;

	struct vm_area_struct *vma;

	struct task_struct *tsk;

	struct mm_struct *mm;

	vm_fault_t fault, major = 0;

	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;

	u32 pkey;



	tsk = current;

	mm = tsk->mm;



	prefetchw(&mm->mmap_sem);



	if (unlikely(kmmio_fault(regs, address)))

		return;



	/*

	 * We fault-in kernel-space virtual memory on-demand. The

	 * We can fault-in kernel-space virtual memory on-demand. The

	 * 'reference' page table is init_mm.pgd.

	 *

	 * NOTE! We MUST NOT take any locks for this case. We may
@@ -1236,29 +1220,68 @@ __do_page_fault(struct pt_regs *regs, unsigned long hw_error_code, 
	 * only copy the information from the master page table,

	 * nothing more.

	 *

	 * This verifies that the fault happens in kernel space

	 * (hw_error_code & 4) == 0, and that the fault was not a

	 * protection error (hw_error_code & 9) == 0.

	 * Before doing this on-demand faulting, ensure that the

	 * fault is not any of the following:

	 * 1. A fault on a PTE with a reserved bit set.

	 * 2. A fault caused by a user-mode access.  (Do not demand-

	 *    fault kernel memory due to user-mode accesses).

	 * 3. A fault caused by a page-level protection violation.

	 *    (A demand fault would be on a non-present page which

	 *     would have X86_PF_PROT==0).

	 */

	if (unlikely(fault_in_kernel_space(address))) {

	if (!(hw_error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {

		if (vmalloc_fault(address) >= 0)

			return;

	}



		/* Can handle a stale RO->RW TLB: */

		if (spurious_fault(hw_error_code, address))

	/* Was the fault spurious, caused by lazy TLB invalidation? */

	if (spurious_kernel_fault(hw_error_code, address))

		return;



	/* kprobes don't want to hook the spurious faults: */

	if (kprobes_fault(regs))

		return;



	/*

	 * Note, despite being a "bad area", there are quite a few

	 * acceptable reasons to get here, such as erratum fixups

	 * and handling kernel code that can fault, like get_user().

	 *

	 * Don't take the mm semaphore here. If we fixup a prefetch

	 * fault we could otherwise deadlock:

	 */

	bad_area_nosemaphore(regs, hw_error_code, address, NULL);

}

NOKPROBE_SYMBOL(do_kern_addr_fault);



/*

 * This routine handles page faults.  It determines the address,

 * and the problem, and then passes it off to one of the appropriate

 * routines.

 */

static noinline void

__do_page_fault(struct pt_regs *regs, unsigned long hw_error_code,

		unsigned long address)

{

	unsigned long sw_error_code;

	struct vm_area_struct *vma;

	struct task_struct *tsk;

	struct mm_struct *mm;

	vm_fault_t fault, major = 0;

	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;

	u32 pkey;



	tsk = current;

	mm = tsk->mm;



	prefetchw(&mm->mmap_sem);



	if (unlikely(kmmio_fault(regs, address)))

		return;



	/* Was the fault on kernel-controlled part of the address space? */

	if (unlikely(fault_in_kernel_space(address))) {

		do_kern_addr_fault(regs, hw_error_code, address);

		return;

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE