Commit b518154e authored by Joonsoo Kim's avatar Joonsoo Kim Committed by Linus Torvalds
Browse files

mm/vmscan: protect the workingset on anonymous LRU 



In current implementation, newly created or swap-in anonymous page is
started on active list.  Growing active list results in rebalancing
active/inactive list so old pages on active list are demoted to inactive
list.  Hence, the page on active list isn't protected at all.

Following is an example of this situation.

Assume that 50 hot pages on active list.  Numbers denote the number of
pages on active/inactive list (active | inactive).

1. 50 hot pages on active list
50(h) | 0

2. workload: 50 newly created (used-once) pages
50(uo) | 50(h)

3. workload: another 50 newly created (used-once) pages
50(uo) | 50(uo), swap-out 50(h)

This patch tries to fix this issue.  Like as file LRU, newly created or
swap-in anonymous pages will be inserted to the inactive list.  They are
promoted to active list if enough reference happens.  This simple
modification changes the above example as following.

1. 50 hot pages on active list
50(h) | 0

2. workload: 50 newly created (used-once) pages
50(h) | 50(uo)

3. workload: another 50 newly created (used-once) pages
50(h) | 50(uo), swap-out 50(uo)

As you can see, hot pages on active list would be protected.

Note that, this implementation has a drawback that the page cannot be
promoted and will be swapped-out if re-access interval is greater than the
size of inactive list but less than the size of total(active+inactive).
To solve this potential issue, following patch will apply workingset
detection similar to the one that's already applied to file LRU.

Signed-off-by: default avatarJoonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Link: http://lkml.kernel.org/r/1595490560-15117-3-git-send-email-iamjoonsoo.kim@lge.com


Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent ccc5dc67

include/linux/swap.h

+1 −1
Original line number Diff line number Diff line
@@ -352,7 +352,7 @@ extern void deactivate_page(struct page *page); 
extern void mark_page_lazyfree(struct page *page);

extern void swap_setup(void);



extern void lru_cache_add_active_or_unevictable(struct page *page,

extern void lru_cache_add_inactive_or_unevictable(struct page *page,

						struct vm_area_struct *vma);



/* linux/mm/vmscan.c */

kernel/events/uprobes.c

+1 −1
Original line number Diff line number Diff line
@@ -184,7 +184,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 
	if (new_page) {

		get_page(new_page);

		page_add_new_anon_rmap(new_page, vma, addr, false);

		lru_cache_add_active_or_unevictable(new_page, vma);

		lru_cache_add_inactive_or_unevictable(new_page, vma);

	} else

		/* no new page, just dec_mm_counter for old_page */

		dec_mm_counter(mm, MM_ANONPAGES);

mm/huge_memory.c

+1 −1
Original line number Diff line number Diff line
@@ -640,7 +640,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, 
		entry = mk_huge_pmd(page, vma->vm_page_prot);

		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);

		page_add_new_anon_rmap(page, vma, haddr, true);

		lru_cache_add_active_or_unevictable(page, vma);

		lru_cache_add_inactive_or_unevictable(page, vma);

		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);

		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);

		add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);

mm/khugepaged.c

+1 −1
Original line number Diff line number Diff line
@@ -1173,7 +1173,7 @@ static void collapse_huge_page(struct mm_struct *mm, 
	spin_lock(pmd_ptl);

	BUG_ON(!pmd_none(*pmd));

	page_add_new_anon_rmap(new_page, vma, address, true);

	lru_cache_add_active_or_unevictable(new_page, vma);

	lru_cache_add_inactive_or_unevictable(new_page, vma);

	pgtable_trans_huge_deposit(mm, pmd, pgtable);

	set_pmd_at(mm, address, pmd, _pmd);

	update_mmu_cache_pmd(vma, address, pmd);

mm/memory.c

+4 −5
Original line number Diff line number Diff line
@@ -2715,7 +2715,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) 
		 */

		ptep_clear_flush_notify(vma, vmf->address, vmf->pte);

		page_add_new_anon_rmap(new_page, vma, vmf->address, false);

		lru_cache_add_active_or_unevictable(new_page, vma);

		lru_cache_add_inactive_or_unevictable(new_page, vma);

		/*

		 * We call the notify macro here because, when using secondary

		 * mmu page tables (such as kvm shadow page tables), we want the
@@ -3266,10 +3266,9 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) 
	/* ksm created a completely new copy */

	if (unlikely(page != swapcache && swapcache)) {

		page_add_new_anon_rmap(page, vma, vmf->address, false);

		lru_cache_add_active_or_unevictable(page, vma);

		lru_cache_add_inactive_or_unevictable(page, vma);

	} else {

		do_page_add_anon_rmap(page, vma, vmf->address, exclusive);

		activate_page(page);

	}



	swap_free(entry);
@@ -3414,7 +3413,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) 


	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);

	page_add_new_anon_rmap(page, vma, vmf->address, false);

	lru_cache_add_active_or_unevictable(page, vma);

	lru_cache_add_inactive_or_unevictable(page, vma);

setpte:

	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
@@ -3672,7 +3671,7 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page) 
	if (write && !(vma->vm_flags & VM_SHARED)) {

		inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);

		page_add_new_anon_rmap(page, vma, vmf->address, false);

		lru_cache_add_active_or_unevictable(page, vma);

		lru_cache_add_inactive_or_unevictable(page, vma);

	} else {

		inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));

		page_add_file_rmap(page, false);

CRA Git | Maintained and supported by SUSTech CRA and CCSE