Commit 4d96ba35 authored by Roman Gushchin's avatar Roman Gushchin Committed by Linus Torvalds
Browse files

mm: memcg/slab: stop setting page->mem_cgroup pointer for slab pages 

Every slab page charged to a non-root memory cgroup has a pointer to the
memory cgroup and holds a reference to it, which protects a non-empty
memory cgroup from being released.  At the same time the page has a
pointer to the corresponding kmem_cache, and also hold a reference to the
kmem_cache.  And kmem_cache by itself holds a reference to the cgroup.

So there is clearly some redundancy, which allows to stop setting the
page->mem_cgroup pointer and rely on getting memcg pointer indirectly via
kmem_cache.  Further it will allow to change this pointer easier, without
a need to go over all charged pages.

So let's stop setting page->mem_cgroup pointer for slab pages, and stop
using the css refcounter directly for protecting the memory cgroup from
going away.  Instead rely on kmem_cache as an intermediate object.

Make sure that vmstats and shrinker lists are working as previously, as
well as /proc/kpagecgroup interface.

Link: http://lkml.kernel.org/r/20190611231813.3148843-10-guro@fb.com


Signed-off-by: default avatarRoman Gushchin <guro@fb.com>
Acked-by: default avatarVladimir Davydov <vdavydov.dev@gmail.com>
Reviewed-by: default avatarShakeel Butt <shakeelb@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Waiman Long <longman@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Andrei Vagin <avagin@gmail.com>
Cc: Qian Cai <cai@lca.pw>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent f0a3a24b

mm/list_lru.c

+2 −1
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
#include <linux/slab.h>

#include <linux/mutex.h>

#include <linux/memcontrol.h>

#include "slab.h"



#ifdef CONFIG_MEMCG_KMEM

static LIST_HEAD(list_lrus);
@@ -63,7 +64,7 @@ static __always_inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr) 
	if (!memcg_kmem_enabled())

		return NULL;

	page = virt_to_head_page(ptr);

	return page->mem_cgroup;

	return memcg_from_slab_page(page);

}



static inline struct list_lru_one *

mm/memcontrol.c

+7 −5
Original line number Diff line number Diff line
@@ -486,6 +486,9 @@ ino_t page_cgroup_ino(struct page *page) 
	unsigned long ino = 0;



	rcu_read_lock();

	if (PageHead(page) && PageSlab(page))

		memcg = memcg_from_slab_page(page);

	else

		memcg = READ_ONCE(page->mem_cgroup);

	while (memcg && !(memcg->css.flags & CSS_ONLINE))

		memcg = parent_mem_cgroup(memcg);
@@ -2802,9 +2805,6 @@ int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, 
		cancel_charge(memcg, nr_pages);

		return -ENOMEM;

	}



	page->mem_cgroup = memcg;



	return 0;

}
@@ -2827,9 +2827,11 @@ int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order) 
	memcg = get_mem_cgroup_from_current();

	if (!mem_cgroup_is_root(memcg)) {

		ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);

		if (!ret)

		if (!ret) {

			page->mem_cgroup = memcg;

			__SetPageKmemcg(page);

		}

	}

	css_put(&memcg->css);

	return ret;

}

mm/slab.h

+61 −13
Original line number Diff line number Diff line
@@ -255,30 +255,67 @@ static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s) 
	return s->memcg_params.root_cache;

}



/*

 * Expects a pointer to a slab page. Please note, that PageSlab() check

 * isn't sufficient, as it returns true also for tail compound slab pages,

 * which do not have slab_cache pointer set.

 * So this function assumes that the page can pass PageHead() and PageSlab()

 * checks.

 */

static inline struct mem_cgroup *memcg_from_slab_page(struct page *page)

{

	struct kmem_cache *s;



	s = READ_ONCE(page->slab_cache);

	if (s && !is_root_cache(s))

		return s->memcg_params.memcg;



	return NULL;

}



/*

 * Charge the slab page belonging to the non-root kmem_cache.

 * Can be called for non-root kmem_caches only.

 */

static __always_inline int memcg_charge_slab(struct page *page,

					     gfp_t gfp, int order,

					     struct kmem_cache *s)

{

	struct mem_cgroup *memcg;

	struct lruvec *lruvec;

	int ret;



	if (is_root_cache(s))

		return 0;



	ret = memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg);

	memcg = s->memcg_params.memcg;

	ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);

	if (ret)

		return ret;



	lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);

	mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order);



	/* transer try_charge() page references to kmem_cache */

	percpu_ref_get_many(&s->memcg_params.refcnt, 1 << order);

	css_put_many(&memcg->css, 1 << order);



	return 0;

}



/*

 * Uncharge a slab page belonging to a non-root kmem_cache.

 * Can be called for non-root kmem_caches only.

 */

static __always_inline void memcg_uncharge_slab(struct page *page, int order,

						struct kmem_cache *s)

{

	if (!is_root_cache(s))

	struct mem_cgroup *memcg;

	struct lruvec *lruvec;



	memcg = s->memcg_params.memcg;

	lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);

	mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order));

	memcg_kmem_uncharge_memcg(page, order, memcg);



	percpu_ref_put_many(&s->memcg_params.refcnt, 1 << order);

	memcg_kmem_uncharge(page, order);

}



extern void slab_init_memcg_params(struct kmem_cache *);
@@ -314,6 +351,11 @@ static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s) 
	return s;

}



static inline struct mem_cgroup *memcg_from_slab_page(struct page *page)

{

	return NULL;

}



static inline int memcg_charge_slab(struct page *page, gfp_t gfp, int order,

				    struct kmem_cache *s)

{
@@ -351,18 +393,24 @@ static __always_inline int charge_slab_page(struct page *page, 
					    gfp_t gfp, int order,

					    struct kmem_cache *s)

{

	int ret = memcg_charge_slab(page, gfp, order, s);



	if (!ret)

		mod_lruvec_page_state(page, cache_vmstat_idx(s), 1 << order);

	if (is_root_cache(s)) {

		mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),

				    1 << order);

		return 0;

	}



	return ret;

	return memcg_charge_slab(page, gfp, order, s);

}



static __always_inline void uncharge_slab_page(struct page *page, int order,

					       struct kmem_cache *s)

{

	mod_lruvec_page_state(page, cache_vmstat_idx(s), -(1 << order));

	if (is_root_cache(s)) {

		mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),

				    -(1 << order));

		return;

	}



	memcg_uncharge_slab(page, order, s);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE