Commit 41bb3476 authored by Cyrill Gorcunov's avatar Cyrill Gorcunov Committed by Linus Torvalds
Browse files

mm: save soft-dirty bits on file pages 



Andy reported that if file page get reclaimed we lose the soft-dirty bit
if it was there, so save _PAGE_BIT_SOFT_DIRTY bit when page address get
encoded into pte entry.  Thus when #pf happens on such non-present pte
we can restore it back.

Reported-by: default avatarAndy Lutomirski <luto@amacapital.net>
Signed-off-by: default avatarCyrill Gorcunov <gorcunov@openvz.org>
Acked-by: default avatarPavel Emelyanov <xemul@parallels.com>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@gmail.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Wanpeng Li <liwanp@linux.vnet.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 179ef71c

arch/x86/include/asm/pgtable-2level.h

+47 −1
Original line number Diff line number Diff line
@@ -55,9 +55,53 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) 
#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)

#endif



#ifdef CONFIG_MEM_SOFT_DIRTY



/*

 * Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE, _PAGE_BIT_SOFT_DIRTY and

 * _PAGE_BIT_PROTNONE are taken, split up the 28 bits of offset

 * into this range.

 */

#define PTE_FILE_MAX_BITS	28

#define PTE_FILE_SHIFT1		(_PAGE_BIT_PRESENT + 1)

#define PTE_FILE_SHIFT2		(_PAGE_BIT_FILE + 1)

#define PTE_FILE_SHIFT3		(_PAGE_BIT_PROTNONE + 1)

#define PTE_FILE_SHIFT4		(_PAGE_BIT_SOFT_DIRTY + 1)

#define PTE_FILE_BITS1		(PTE_FILE_SHIFT2 - PTE_FILE_SHIFT1 - 1)

#define PTE_FILE_BITS2		(PTE_FILE_SHIFT3 - PTE_FILE_SHIFT2 - 1)

#define PTE_FILE_BITS3		(PTE_FILE_SHIFT4 - PTE_FILE_SHIFT3 - 1)



#define pte_to_pgoff(pte)						\

	((((pte).pte_low >> (PTE_FILE_SHIFT1))				\

	  & ((1U << PTE_FILE_BITS1) - 1)))				\

	+ ((((pte).pte_low >> (PTE_FILE_SHIFT2))			\

	    & ((1U << PTE_FILE_BITS2) - 1))				\

	   << (PTE_FILE_BITS1))						\

	+ ((((pte).pte_low >> (PTE_FILE_SHIFT3))			\

	    & ((1U << PTE_FILE_BITS3) - 1))				\

	   << (PTE_FILE_BITS1 + PTE_FILE_BITS2))			\

	+ ((((pte).pte_low >> (PTE_FILE_SHIFT4)))			\

	    << (PTE_FILE_BITS1 + PTE_FILE_BITS2 + PTE_FILE_BITS3))



#define pgoff_to_pte(off)						\

	((pte_t) { .pte_low =						\

	 ((((off)) & ((1U << PTE_FILE_BITS1) - 1)) << PTE_FILE_SHIFT1)	\

	 + ((((off) >> PTE_FILE_BITS1)					\

	     & ((1U << PTE_FILE_BITS2) - 1))				\

	    << PTE_FILE_SHIFT2)						\

	 + ((((off) >> (PTE_FILE_BITS1 + PTE_FILE_BITS2))		\

	     & ((1U << PTE_FILE_BITS3) - 1))				\

	    << PTE_FILE_SHIFT3)						\

	 + ((((off) >>							\

	      (PTE_FILE_BITS1 + PTE_FILE_BITS2 + PTE_FILE_BITS3)))	\

	    << PTE_FILE_SHIFT4)						\

	 + _PAGE_FILE })



#else /* CONFIG_MEM_SOFT_DIRTY */



/*

 * Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE and _PAGE_BIT_PROTNONE are taken,

 * split up the 29 bits of offset into this range:

 * split up the 29 bits of offset into this range.

 */

#define PTE_FILE_MAX_BITS	29

#define PTE_FILE_SHIFT1		(_PAGE_BIT_PRESENT + 1)
@@ -88,6 +132,8 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) 
	    << PTE_FILE_SHIFT3)						\

	 + _PAGE_FILE })



#endif /* CONFIG_MEM_SOFT_DIRTY */



/* Encode and de-code a swap entry */

#if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE

#define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1)

arch/x86/include/asm/pgtable-3level.h

+3 −0
Original line number Diff line number Diff line
@@ -179,6 +179,9 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp) 
/*

 * Bits 0, 6 and 7 are taken in the low part of the pte,

 * put the 32 bits of offset into the high part.

 *

 * For soft-dirty tracking 11 bit is taken from

 * the low part of pte as well.

 */

#define pte_to_pgoff(pte) ((pte).pte_high)

#define pgoff_to_pte(off)						\

arch/x86/include/asm/pgtable.h

+15 −0
Original line number Diff line number Diff line
@@ -329,6 +329,21 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) 
	return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);

}



static inline pte_t pte_file_clear_soft_dirty(pte_t pte)

{

	return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);

}



static inline pte_t pte_file_mksoft_dirty(pte_t pte)

{

	return pte_set_flags(pte, _PAGE_SOFT_DIRTY);

}



static inline int pte_file_soft_dirty(pte_t pte)

{

	return pte_flags(pte) & _PAGE_SOFT_DIRTY;

}



/*

 * Mask out unsupported bits in a present pgprot.  Non-present pgprots

 * can use those bits for other purposes, so leave them be.

arch/x86/include/asm/pgtable_types.h

+3 −1
Original line number Diff line number Diff line
@@ -61,8 +61,10 @@ 
 * they do not conflict with each other.

 */



#define _PAGE_BIT_SOFT_DIRTY	_PAGE_BIT_HIDDEN



#ifdef CONFIG_MEM_SOFT_DIRTY

#define _PAGE_SOFT_DIRTY	(_AT(pteval_t, 1) << _PAGE_BIT_HIDDEN)

#define _PAGE_SOFT_DIRTY	(_AT(pteval_t, 1) << _PAGE_BIT_SOFT_DIRTY)

#else

#define _PAGE_SOFT_DIRTY	(_AT(pteval_t, 0))

#endif

fs/proc/task_mmu.c

+2 −0
Original line number Diff line number Diff line
@@ -736,6 +736,8 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma, 
		ptent = pte_clear_flags(ptent, _PAGE_SOFT_DIRTY);

	} else if (is_swap_pte(ptent)) {

		ptent = pte_swp_clear_soft_dirty(ptent);

	} else if (pte_file(ptent)) {

		ptent = pte_file_clear_soft_dirty(ptent);

	}



	set_pte_at(vma->vm_mm, addr, pte, ptent);

CRA Git | Maintained and supported by SUSTech CRA and CCSE