Commit be5d0a74 authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds
Browse files

mm: memcontrol: switch to native NR_ANON_MAPPED counter 



Memcg maintains a private MEMCG_RSS counter.  This divergence from the
generic VM accounting means unnecessary code overhead, and creates a
dependency for memcg that page->mapping is set up at the time of charging,
so that page types can be told apart.

Convert the generic accounting sites to mod_lruvec_page_state and friends
to maintain the per-cgroup vmstat counter of NR_ANON_MAPPED.  We use
lock_page_memcg() to stabilize page->mem_cgroup during rmap changes, the
same way we do for NR_FILE_MAPPED.

With the previous patch removing MEMCG_CACHE and the private NR_SHMEM
counter, this patch finally eliminates the need to have page->mapping set
up at charge time.  However, we need to have page->mem_cgroup set up by
the time rmap runs and does the accounting, so switch the commit and the
rmap callbacks around.

v2: fix temporary accounting bug by switching rmap<->commit (Joonsoo)

Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Cc: Alex Shi <alex.shi@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Roman Gushchin <guro@fb.com>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Balbir Singh <bsingharora@gmail.com>
Link: http://lkml.kernel.org/r/20200508183105.225460-11-hannes@cmpxchg.org


Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 0d1c2072

include/linux/memcontrol.h

+1 −2
Original line number Diff line number Diff line
@@ -29,8 +29,7 @@ struct kmem_cache; 


/* Cgroup-specific page state, on top of universal node page state */

enum memcg_stat_item {

	MEMCG_RSS = NR_VM_NODE_STAT_ITEMS,

	MEMCG_RSS_HUGE,

	MEMCG_RSS_HUGE = NR_VM_NODE_STAT_ITEMS,

	MEMCG_SWAP,

	MEMCG_SOCK,

	/* XXX: why are these zone and not node counters? */

kernel/events/uprobes.c

+1 −1
Original line number Diff line number Diff line
@@ -188,8 +188,8 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 


	if (new_page) {

		get_page(new_page);

		page_add_new_anon_rmap(new_page, vma, addr, false);

		mem_cgroup_commit_charge(new_page, memcg, false);

		page_add_new_anon_rmap(new_page, vma, addr, false);

		lru_cache_add_active_or_unevictable(new_page, vma);

	} else

		/* no new page, just dec_mm_counter for old_page */

mm/huge_memory.c

+1 −1
Original line number Diff line number Diff line
@@ -640,8 +640,8 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, 


		entry = mk_huge_pmd(page, vma->vm_page_prot);

		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);

		page_add_new_anon_rmap(page, vma, haddr, true);

		mem_cgroup_commit_charge(page, memcg, false);

		page_add_new_anon_rmap(page, vma, haddr, true);

		lru_cache_add_active_or_unevictable(page, vma);

		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);

		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);

mm/khugepaged.c

+1 −1
Original line number Diff line number Diff line
@@ -1175,8 +1175,8 @@ static void collapse_huge_page(struct mm_struct *mm, 


	spin_lock(pmd_ptl);

	BUG_ON(!pmd_none(*pmd));

	page_add_new_anon_rmap(new_page, vma, address, true);

	mem_cgroup_commit_charge(new_page, memcg, false);

	page_add_new_anon_rmap(new_page, vma, address, true);

	count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);

	lru_cache_add_active_or_unevictable(new_page, vma);

	pgtable_trans_huge_deposit(mm, pmd, pgtable);

mm/memcontrol.c

+9 −18
Original line number Diff line number Diff line
@@ -836,13 +836,6 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 
					 struct page *page,

					 int nr_pages)

{

	/*

	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is

	 * counted as CACHE even if it's on ANON LRU.

	 */

	if (PageAnon(page))

		__mod_memcg_state(memcg, MEMCG_RSS, nr_pages);



	if (abs(nr_pages) > 1) {

		VM_BUG_ON_PAGE(!PageTransHuge(page), page);

		__mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
@@ -1384,7 +1377,7 @@ static char *memory_stat_format(struct mem_cgroup *memcg) 
	 */



	seq_buf_printf(&s, "anon %llu\n",

		       (u64)memcg_page_state(memcg, MEMCG_RSS) *

		       (u64)memcg_page_state(memcg, NR_ANON_MAPPED) *

		       PAGE_SIZE);

	seq_buf_printf(&s, "file %llu\n",

		       (u64)memcg_page_state(memcg, NR_FILE_PAGES) *
@@ -3353,7 +3346,7 @@ static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 


	if (mem_cgroup_is_root(memcg)) {

		val = memcg_page_state(memcg, NR_FILE_PAGES) +

			memcg_page_state(memcg, MEMCG_RSS);

			memcg_page_state(memcg, NR_ANON_MAPPED);

		if (swap)

			val += memcg_page_state(memcg, MEMCG_SWAP);

	} else {
@@ -3824,7 +3817,7 @@ static int memcg_numa_stat_show(struct seq_file *m, void *v) 


static const unsigned int memcg1_stats[] = {

	NR_FILE_PAGES,

	MEMCG_RSS,

	NR_ANON_MAPPED,

	MEMCG_RSS_HUGE,

	NR_SHMEM,

	NR_FILE_MAPPED,
@@ -5455,7 +5448,12 @@ static int mem_cgroup_move_account(struct page *page, 


	lock_page_memcg(page);



	if (!PageAnon(page)) {

	if (PageAnon(page)) {

		if (page_mapped(page)) {

			__mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);

			__mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);

		}

	} else {

		__mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);

		__mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
@@ -6589,7 +6587,6 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, 
{

	unsigned int nr_pages = hpage_nr_pages(page);



	VM_BUG_ON_PAGE(!page->mapping, page);

	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);



	if (mem_cgroup_disabled())
@@ -6662,8 +6659,6 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, 
	struct mem_cgroup *memcg;

	int ret;



	VM_BUG_ON_PAGE(!page->mapping, page);



	ret = mem_cgroup_try_charge(page, mm, gfp_mask, &memcg);

	if (ret)

		return ret;
@@ -6675,7 +6670,6 @@ struct uncharge_gather { 
	struct mem_cgroup *memcg;

	unsigned long nr_pages;

	unsigned long pgpgout;

	unsigned long nr_anon;

	unsigned long nr_kmem;

	unsigned long nr_huge;

	struct page *dummy_page;
@@ -6700,7 +6694,6 @@ static void uncharge_batch(const struct uncharge_gather *ug) 
	}



	local_irq_save(flags);

	__mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);

	__mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);

	__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);

	__this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
@@ -6740,8 +6733,6 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) 
	if (!PageKmemcg(page)) {

		if (PageTransHuge(page))

			ug->nr_huge += nr_pages;

		if (PageAnon(page))

			ug->nr_anon += nr_pages;

		ug->pgpgout++;

	} else {

		ug->nr_kmem += nr_pages;

CRA Git | Maintained and supported by SUSTech CRA and CCSE