Commit bba4ed01 authored by Tom Lendacky's avatar Tom Lendacky Committed by Ingo Molnar
Browse files

x86/mm, kexec: Allow kexec to be used with SME 



Provide support so that kexec can be used to boot a kernel when SME is
enabled.

Support is needed to allocate pages for kexec without encryption.  This
is needed in order to be able to reboot in the kernel in the same manner
as originally booted.

Additionally, when shutting down all of the CPUs we need to be sure to
flush the caches and then halt. This is needed when booting from a state
where SME was not active into a state where SME is active (or vice-versa).
Without these steps, it is possible for cache lines to exist for the same
physical location but tagged both with and without the encryption bit. This
can cause random memory corruption when caches are flushed depending on
which cacheline is written last.

Signed-off-by: default avatarTom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarBorislav Petkov <bp@suse.de>
Cc: <kexec@lists.infradead.org>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brijesh Singh <brijesh.singh@amd.com>
Cc: Dave Young <dyoung@redhat.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Matt Fleming <matt@codeblueprint.co.uk>
Cc: Michael S. Tsirkin <mst@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Radim Krčmář <rkrcmar@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Toshimitsu Kani <toshi.kani@hpe.com>
Cc: kasan-dev@googlegroups.com
Cc: kvm@vger.kernel.org
Cc: linux-arch@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: linux-efi@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/b95ff075db3e7cd545313f2fb609a49619a09625.1500319216.git.thomas.lendacky@amd.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent d0ec49d4

arch/x86/include/asm/init.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -7,6 +7,7 @@ struct x86_mapping_info { 
	unsigned long page_flag;	 /* page flag for PMD or PUD entry */

	unsigned long page_flag;	 /* page flag for PMD or PUD entry */

	unsigned long offset;		 /* ident mapping offset */

	unsigned long offset;		 /* ident mapping offset */

	bool direct_gbpages;		 /* PUD level 1GB page support */

	bool direct_gbpages;		 /* PUD level 1GB page support */

	unsigned long kernpg_flag;	 /* kernel pagetable flag override */

};

};





int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,

int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,

arch/x86/include/asm/kexec.h

+8 −0
Original line number Original line Diff line number Diff line
@@ -207,6 +207,14 @@ struct kexec_entry64_regs { 
	uint64_t r15;

	uint64_t r15;

	uint64_t rip;

	uint64_t rip;

};

};



extern int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages,

				       gfp_t gfp);

#define arch_kexec_post_alloc_pages arch_kexec_post_alloc_pages



extern void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages);

#define arch_kexec_pre_free_pages arch_kexec_pre_free_pages



#endif

#endif





typedef void crash_vmclear_fn(void);

typedef void crash_vmclear_fn(void);

arch/x86/include/asm/pgtable_types.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -213,6 +213,7 @@ enum page_cache_mode { 
#define PAGE_KERNEL		__pgprot(__PAGE_KERNEL | _PAGE_ENC)

#define PAGE_KERNEL		__pgprot(__PAGE_KERNEL | _PAGE_ENC)

#define PAGE_KERNEL_RO		__pgprot(__PAGE_KERNEL_RO | _PAGE_ENC)

#define PAGE_KERNEL_RO		__pgprot(__PAGE_KERNEL_RO | _PAGE_ENC)

#define PAGE_KERNEL_EXEC	__pgprot(__PAGE_KERNEL_EXEC | _PAGE_ENC)

#define PAGE_KERNEL_EXEC	__pgprot(__PAGE_KERNEL_EXEC | _PAGE_ENC)

#define PAGE_KERNEL_EXEC_NOENC	__pgprot(__PAGE_KERNEL_EXEC)

#define PAGE_KERNEL_RX		__pgprot(__PAGE_KERNEL_RX | _PAGE_ENC)

#define PAGE_KERNEL_RX		__pgprot(__PAGE_KERNEL_RX | _PAGE_ENC)

#define PAGE_KERNEL_NOCACHE	__pgprot(__PAGE_KERNEL_NOCACHE | _PAGE_ENC)

#define PAGE_KERNEL_NOCACHE	__pgprot(__PAGE_KERNEL_NOCACHE | _PAGE_ENC)

#define PAGE_KERNEL_LARGE	__pgprot(__PAGE_KERNEL_LARGE | _PAGE_ENC)

#define PAGE_KERNEL_LARGE	__pgprot(__PAGE_KERNEL_LARGE | _PAGE_ENC)

arch/x86/kernel/machine_kexec_64.c

+21 −1
Original line number Original line Diff line number Diff line
@@ -87,7 +87,7 @@ static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) 
		set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));

		set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));

	}

	}

	pte = pte_offset_kernel(pmd, vaddr);

	pte = pte_offset_kernel(pmd, vaddr);

	set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));

	set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC_NOENC));

	return 0;

	return 0;

err:

err:

	free_transition_pgtable(image);

	free_transition_pgtable(image);
@@ -115,6 +115,7 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable) 
		.alloc_pgt_page	= alloc_pgt_page,

		.alloc_pgt_page	= alloc_pgt_page,

		.context	= image,

		.context	= image,

		.page_flag	= __PAGE_KERNEL_LARGE_EXEC,

		.page_flag	= __PAGE_KERNEL_LARGE_EXEC,

		.kernpg_flag	= _KERNPG_TABLE_NOENC,

	};

	};

	unsigned long mstart, mend;

	unsigned long mstart, mend;

	pgd_t *level4p;

	pgd_t *level4p;
@@ -602,3 +603,22 @@ void arch_kexec_unprotect_crashkres(void) 
{

{

	kexec_mark_crashkres(false);

	kexec_mark_crashkres(false);

}

}



int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp)

{

	/*

	 * If SME is active we need to be sure that kexec pages are

	 * not encrypted because when we boot to the new kernel the

	 * pages won't be accessed encrypted (initially).

	 */

	return set_memory_decrypted((unsigned long)vaddr, pages);

}



void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages)

{

	/*

	 * If SME is active we need to reset the pages back to being

	 * an encrypted mapping before freeing them.

	 */

	set_memory_encrypted((unsigned long)vaddr, pages);

}

arch/x86/kernel/process.c

+15 −2
Original line number Original line Diff line number Diff line
@@ -355,6 +355,7 @@ bool xen_set_default_idle(void) 
	return ret;

	return ret;

}

}

#endif

#endif



void stop_this_cpu(void *dummy)

void stop_this_cpu(void *dummy)

{

{

	local_irq_disable();

	local_irq_disable();
@@ -365,8 +366,20 @@ void stop_this_cpu(void *dummy) 
	disable_local_APIC();

	disable_local_APIC();

	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));

	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));





	for (;;)

	for (;;) {

		halt();

		/*

		 * Use wbinvd followed by hlt to stop the processor. This

		 * provides support for kexec on a processor that supports

		 * SME. With kexec, going from SME inactive to SME active

		 * requires clearing cache entries so that addresses without

		 * the encryption bit set don't corrupt the same physical

		 * address that has the encryption bit set when caches are

		 * flushed. To achieve this a wbinvd is performed followed by

		 * a hlt. Even if the processor is not in the kexec/SME

		 * scenario this only adds a wbinvd to a halting processor.

		 */

		asm volatile("wbinvd; hlt" : : : "memory");

	}

}

}





/*

/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE