Commit dd78fedd authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds
Browse files

rmap: support file thp 

Naive approach: on mapping/unmapping the page as compound we update
->_mapcount on each 4k page.  That's not efficient, but it's not obvious
how we can optimize this.  We can look into optimization later.

PG_double_map optimization doesn't work for file pages since lifecycle
of file pages is different comparing to anon pages: file page can be
mapped again at any time.

Link: http://lkml.kernel.org/r/1466021202-61880-11-git-send-email-kirill.shutemov@linux.intel.com


Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 7267ec00

include/linux/rmap.h

+1 −1
Original line number Diff line number Diff line
@@ -165,7 +165,7 @@ void do_page_add_anon_rmap(struct page *, struct vm_area_struct *, 
			   unsigned long, int);

void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,

		unsigned long, bool);

void page_add_file_rmap(struct page *);

void page_add_file_rmap(struct page *, bool);

void page_remove_rmap(struct page *, bool);



void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,

mm/huge_memory.c

+7 −3
Original line number Diff line number Diff line
@@ -3301,18 +3301,22 @@ static void __split_huge_page(struct page *page, struct list_head *list) 


int total_mapcount(struct page *page)

{

	int i, ret;

	int i, compound, ret;



	VM_BUG_ON_PAGE(PageTail(page), page);



	if (likely(!PageCompound(page)))

		return atomic_read(&page->_mapcount) + 1;



	ret = compound_mapcount(page);

	compound = compound_mapcount(page);

	if (PageHuge(page))

		return ret;

		return compound;

	ret = compound;

	for (i = 0; i < HPAGE_PMD_NR; i++)

		ret += atomic_read(&page[i]._mapcount) + 1;

	/* File pages has compound_mapcount included in _mapcount */

	if (!PageAnon(page))

		return ret - compound * HPAGE_PMD_NR;

	if (PageDoubleMap(page))

		ret -= HPAGE_PMD_NR;

	return ret;

mm/memory.c

+2 −2
Original line number Diff line number Diff line
@@ -1494,7 +1494,7 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, 
	/* Ok, finally just insert the thing.. */

	get_page(page);

	inc_mm_counter_fast(mm, mm_counter_file(page));

	page_add_file_rmap(page);

	page_add_file_rmap(page, false);

	set_pte_at(mm, addr, pte, mk_pte(page, prot));



	retval = 0;
@@ -2962,7 +2962,7 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg, 
		lru_cache_add_active_or_unevictable(page, vma);

	} else {

		inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));

		page_add_file_rmap(page);

		page_add_file_rmap(page, false);

	}

	set_pte_at(vma->vm_mm, fe->address, fe->pte, entry);

mm/migrate.c

+1 −1
Original line number Diff line number Diff line
@@ -259,7 +259,7 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, 
	} else if (PageAnon(new))

		page_add_anon_rmap(new, vma, addr, false);

	else

		page_add_file_rmap(new);

		page_add_file_rmap(new, false);



	if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))

		mlock_vma_page(new);

mm/rmap.c

+35 −13
Original line number Diff line number Diff line
@@ -1270,18 +1270,34 @@ void page_add_new_anon_rmap(struct page *page, 
 *

 * The caller needs to hold the pte lock.

 */

void page_add_file_rmap(struct page *page)

void page_add_file_rmap(struct page *page, bool compound)

{

	int i, nr = 1;



	VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);

	lock_page_memcg(page);

	if (atomic_inc_and_test(&page->_mapcount)) {

		__inc_zone_page_state(page, NR_FILE_MAPPED);

		mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);

	if (compound && PageTransHuge(page)) {

		for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {

			if (atomic_inc_and_test(&page[i]._mapcount))

				nr++;

		}

		if (!atomic_inc_and_test(compound_mapcount_ptr(page)))

			goto out;

	} else {

		if (!atomic_inc_and_test(&page->_mapcount))

			goto out;

	}

	__mod_zone_page_state(page_zone(page), NR_FILE_MAPPED, nr);

	mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);

out:

	unlock_page_memcg(page);

}



static void page_remove_file_rmap(struct page *page)

static void page_remove_file_rmap(struct page *page, bool compound)

{

	int i, nr = 1;



	VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);

	lock_page_memcg(page);



	/* Hugepages are not counted in NR_FILE_MAPPED for now. */
@@ -1292,15 +1308,24 @@ static void page_remove_file_rmap(struct page *page) 
	}



	/* page still mapped by someone else? */

	if (compound && PageTransHuge(page)) {

		for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {

			if (atomic_add_negative(-1, &page[i]._mapcount))

				nr++;

		}

		if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))

			goto out;

	} else {

		if (!atomic_add_negative(-1, &page->_mapcount))

			goto out;

	}



	/*

	 * We use the irq-unsafe __{inc|mod}_zone_page_stat because

	 * these counters are not modified in interrupt context, and

	 * pte lock(a spinlock) is held, which implies preemption disabled.

	 */

	__dec_zone_page_state(page, NR_FILE_MAPPED);

	__mod_zone_page_state(page_zone(page), NR_FILE_MAPPED, -nr);

	mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);



	if (unlikely(PageMlocked(page)))
@@ -1356,11 +1381,8 @@ static void page_remove_anon_compound_rmap(struct page *page) 
 */

void page_remove_rmap(struct page *page, bool compound)

{

	if (!PageAnon(page)) {

		VM_BUG_ON_PAGE(compound && !PageHuge(page), page);

		page_remove_file_rmap(page);

		return;

	}

	if (!PageAnon(page))

		return page_remove_file_rmap(page, compound);



	if (compound)

		return page_remove_anon_compound_rmap(page);

CRA Git | Maintained and supported by SUSTech CRA and CCSE