Commit 36943aba authored by Steven Price's avatar Steven Price Committed by Catalin Marinas
Browse files

arm64: mte: Enable swap of tagged pages 



When swapping pages out to disk it is necessary to save any tags that
have been set, and restore when swapping back in. Make use of the new
page flag (PG_ARCH_2, locally named PG_mte_tagged) to identify pages
with tags. When swapping out these pages the tags are stored in memory
and later restored when the pages are brought back in. Because shmem can
swap pages back in without restoring the userspace PTE it is also
necessary to add a hook for shmem.

Signed-off-by: default avatarSteven Price <steven.price@arm.com>
[catalin.marinas@arm.com: move function prototypes to mte.h]
[catalin.marinas@arm.com: drop '_tags' from arch_swap_restore_tags()]
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Will Deacon <will@kernel.org>
parent 8a84802e

arch/arm64/include/asm/mte.h

+8 −0
Original line number Diff line number Diff line
@@ -21,6 +21,14 @@ unsigned long mte_copy_tags_from_user(void *to, const void __user *from, 
				      unsigned long n);

unsigned long mte_copy_tags_to_user(void __user *to, void *from,

				    unsigned long n);

int mte_save_tags(struct page *page);

void mte_save_page_tags(const void *page_addr, void *tag_storage);

bool mte_restore_tags(swp_entry_t entry, struct page *page);

void mte_restore_page_tags(void *page_addr, const void *tag_storage);

void mte_invalidate_tags(int type, pgoff_t offset);

void mte_invalidate_tags_area(int type);

void *mte_allocate_tag_storage(void);

void mte_free_tag_storage(char *storage);



#ifdef CONFIG_ARM64_MTE

arch/arm64/include/asm/pgtable.h

+32 −0
Original line number Diff line number Diff line
@@ -867,6 +867,38 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, 


extern int kern_addr_valid(unsigned long addr);



#ifdef CONFIG_ARM64_MTE



#define __HAVE_ARCH_PREPARE_TO_SWAP

static inline int arch_prepare_to_swap(struct page *page)

{

	if (system_supports_mte())

		return mte_save_tags(page);

	return 0;

}



#define __HAVE_ARCH_SWAP_INVALIDATE

static inline void arch_swap_invalidate_page(int type, pgoff_t offset)

{

	if (system_supports_mte())

		mte_invalidate_tags(type, offset);

}



static inline void arch_swap_invalidate_area(int type)

{

	if (system_supports_mte())

		mte_invalidate_tags_area(type);

}



#define __HAVE_ARCH_SWAP_RESTORE

static inline void arch_swap_restore(swp_entry_t entry, struct page *page)

{

	if (system_supports_mte() && mte_restore_tags(entry, page))

		set_bit(PG_mte_tagged, &page->flags);

}



#endif /* CONFIG_ARM64_MTE */



/*

 * On AArch64, the cache coherency is handled via the set_pte_at() function.

 */

arch/arm64/kernel/mte.c

+18 −1
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ 
#include <linux/sched.h>

#include <linux/sched/mm.h>

#include <linux/string.h>

#include <linux/swap.h>

#include <linux/swapops.h>

#include <linux/thread_info.h>

#include <linux/uio.h>
@@ -18,15 +20,30 @@ 
#include <asm/ptrace.h>

#include <asm/sysreg.h>



static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap)

{

	pte_t old_pte = READ_ONCE(*ptep);



	if (check_swap && is_swap_pte(old_pte)) {

		swp_entry_t entry = pte_to_swp_entry(old_pte);



		if (!non_swap_entry(entry) && mte_restore_tags(entry, page))

			return;

	}



	mte_clear_page_tags(page_address(page));

}



void mte_sync_tags(pte_t *ptep, pte_t pte)

{

	struct page *page = pte_page(pte);

	long i, nr_pages = compound_nr(page);

	bool check_swap = nr_pages == 1;



	/* if PG_mte_tagged is set, tags have already been initialised */

	for (i = 0; i < nr_pages; i++, page++) {

		if (!test_and_set_bit(PG_mte_tagged, &page->flags))

			mte_clear_page_tags(page_address(page));

			mte_sync_page_tags(page, ptep, check_swap);

	}

}

arch/arm64/lib/mte.S

+45 −0
Original line number Diff line number Diff line
@@ -104,3 +104,48 @@ SYM_FUNC_START(mte_copy_tags_to_user) 
2:	sub	x0, x0, x3		// update the number of tags copied

	ret

SYM_FUNC_END(mte_copy_tags_to_user)



/*

 * Save the tags in a page

 *   x0 - page address

 *   x1 - tag storage

 */

SYM_FUNC_START(mte_save_page_tags)

	multitag_transfer_size x7, x5

1:

	mov	x2, #0

2:

	ldgm	x5, [x0]

	orr	x2, x2, x5

	add	x0, x0, x7

	tst	x0, #0xFF		// 16 tag values fit in a register,

	b.ne	2b			// which is 16*16=256 bytes



	str	x2, [x1], #8



	tst	x0, #(PAGE_SIZE - 1)

	b.ne	1b



	ret

SYM_FUNC_END(mte_save_page_tags)



/*

 * Restore the tags in a page

 *   x0 - page address

 *   x1 - tag storage

 */

SYM_FUNC_START(mte_restore_page_tags)

	multitag_transfer_size x7, x5

1:

	ldr	x2, [x1], #8

2:

	stgm	x2, [x0]

	add	x0, x0, x7

	tst	x0, #0xFF

	b.ne	2b



	tst	x0, #(PAGE_SIZE - 1)

	b.ne	1b



	ret

SYM_FUNC_END(mte_restore_page_tags)

arch/arm64/mm/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ obj-$(CONFIG_PTDUMP_CORE) += dump.o 
obj-$(CONFIG_PTDUMP_DEBUGFS)	+= ptdump_debugfs.o

obj-$(CONFIG_NUMA)		+= numa.o

obj-$(CONFIG_DEBUG_VIRTUAL)	+= physaddr.o

obj-$(CONFIG_ARM64_MTE)		+= mteswap.o

KASAN_SANITIZE_physaddr.o	+= n



obj-$(CONFIG_KASAN)		+= kasan_init.o

CRA Git | Maintained and supported by SUSTech CRA and CCSE