Commit 43486694 authored by Roman Gushchin's avatar Roman Gushchin Committed by Linus Torvalds
Browse files

mm: memcg/slab: generalize postponed non-root kmem_cache deactivation 

Currently SLUB uses a work scheduled after an RCU grace period to
deactivate a non-root kmem_cache.  This mechanism can be reused for
kmem_caches release, but requires generalization for SLAB case.

Introduce kmemcg_cache_deactivate() function, which calls
allocator-specific __kmem_cache_deactivate() and schedules execution of
__kmem_cache_deactivate_after_rcu() with all necessary locks in a worker
context after an rcu grace period.

Here is the new calling scheme:
  kmemcg_cache_deactivate()
    __kmemcg_cache_deactivate()                  SLAB/SLUB-specific
    kmemcg_rcufn()                               rcu
      kmemcg_workfn()                            work
        __kmemcg_cache_deactivate_after_rcu()    SLAB/SLUB-specific

instead of:
  __kmemcg_cache_deactivate()                    SLAB/SLUB-specific
    slab_deactivate_memcg_cache_rcu_sched()      SLUB-only
      kmemcg_rcufn()                             rcu
        kmemcg_workfn()                          work
          kmemcg_cache_deact_after_rcu()         SLUB-only

For consistency, all allocator-specific functions start with "__".

Link: http://lkml.kernel.org/r/20190611231813.3148843-4-guro@fb.com


Signed-off-by: default avatarRoman Gushchin <guro@fb.com>
Acked-by: default avatarVladimir Davydov <vdavydov.dev@gmail.com>
Reviewed-by: default avatarShakeel Butt <shakeelb@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Waiman Long <longman@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Andrei Vagin <avagin@gmail.com>
Cc: Qian Cai <cai@lca.pw>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 0b14e8aa

mm/slab.c

+4 −0
Original line number Original line Diff line number Diff line
@@ -2252,6 +2252,10 @@ void __kmemcg_cache_deactivate(struct kmem_cache *cachep) 
{

{

	__kmem_cache_shrink(cachep);

	__kmem_cache_shrink(cachep);

}

}



void __kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s)

{

}

#endif

#endif





int __kmem_cache_shutdown(struct kmem_cache *cachep)

int __kmem_cache_shutdown(struct kmem_cache *cachep)

mm/slab.h

+1 −2
Original line number Original line Diff line number Diff line
@@ -172,6 +172,7 @@ int __kmem_cache_shutdown(struct kmem_cache *); 
void __kmem_cache_release(struct kmem_cache *);

void __kmem_cache_release(struct kmem_cache *);

int __kmem_cache_shrink(struct kmem_cache *);

int __kmem_cache_shrink(struct kmem_cache *);

void __kmemcg_cache_deactivate(struct kmem_cache *s);

void __kmemcg_cache_deactivate(struct kmem_cache *s);

void __kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s);

void slab_kmem_cache_release(struct kmem_cache *);

void slab_kmem_cache_release(struct kmem_cache *);





struct seq_file;

struct seq_file;
@@ -290,8 +291,6 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order, 




extern void slab_init_memcg_params(struct kmem_cache *);

extern void slab_init_memcg_params(struct kmem_cache *);

extern void memcg_link_cache(struct kmem_cache *s, struct mem_cgroup *memcg);

extern void memcg_link_cache(struct kmem_cache *s, struct mem_cgroup *memcg);

extern void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s,

				void (*work_fn)(struct kmem_cache *));





#else /* CONFIG_MEMCG_KMEM */

#else /* CONFIG_MEMCG_KMEM */

mm/slab_common.c

+8 −19
Original line number Original line Diff line number Diff line
@@ -708,7 +708,7 @@ static void kmemcg_workfn(struct work_struct *work) 
	put_online_mems();

	put_online_mems();

	put_online_cpus();

	put_online_cpus();





	/* done, put the ref from slab_deactivate_memcg_cache_rcu_sched() */

	/* done, put the ref from kmemcg_cache_deactivate() */

	css_put(&s->memcg_params.memcg->css);

	css_put(&s->memcg_params.memcg->css);

}

}
@@ -726,31 +726,21 @@ static void kmemcg_rcufn(struct rcu_head *head) 
	queue_work(memcg_kmem_cache_wq, &s->memcg_params.work);

	queue_work(memcg_kmem_cache_wq, &s->memcg_params.work);

}

}





/**

static void kmemcg_cache_deactivate(struct kmem_cache *s)

 * slab_deactivate_memcg_cache_rcu_sched - schedule deactivation after a

 *					   sched RCU grace period

 * @s: target kmem_cache

 * @work_fn: deactivation function to call

 *

 * Schedule @work_fn to be invoked with online cpus, mems and slab_mutex

 * held after a sched RCU grace period.  The slab is guaranteed to stay

 * alive until @work_fn is finished.  This is to be used from

 * __kmemcg_cache_deactivate().

 */

void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s,

					   void (*work_fn)(struct kmem_cache *))

{

{

	if (WARN_ON_ONCE(is_root_cache(s)) ||

	if (WARN_ON_ONCE(is_root_cache(s)) ||

	    WARN_ON_ONCE(s->memcg_params.work_fn))

	    WARN_ON_ONCE(s->memcg_params.work_fn))

		return;

		return;





	__kmemcg_cache_deactivate(s);



	if (s->memcg_params.root_cache->memcg_params.dying)

	if (s->memcg_params.root_cache->memcg_params.dying)

		return;

		return;





	/* pin memcg so that @s doesn't get destroyed in the middle */

	/* pin memcg so that @s doesn't get destroyed in the middle */

	css_get(&s->memcg_params.memcg->css);

	css_get(&s->memcg_params.memcg->css);





	s->memcg_params.work_fn = work_fn;

	s->memcg_params.work_fn = __kmemcg_cache_deactivate_after_rcu;

	call_rcu(&s->memcg_params.rcu_head, kmemcg_rcufn);

	call_rcu(&s->memcg_params.rcu_head, kmemcg_rcufn);

}

}
@@ -773,7 +763,7 @@ void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg) 
		if (!c)

		if (!c)

			continue;

			continue;





		__kmemcg_cache_deactivate(c);

		kmemcg_cache_deactivate(c);

		arr->entries[idx] = NULL;

		arr->entries[idx] = NULL;

	}

	}

	mutex_unlock(&slab_mutex);

	mutex_unlock(&slab_mutex);
@@ -866,10 +856,9 @@ static void flush_memcg_workqueue(struct kmem_cache *s) 
	mutex_unlock(&slab_mutex);

	mutex_unlock(&slab_mutex);





	/*

	/*

	 * SLUB deactivates the kmem_caches through call_rcu. Make

	 * SLAB and SLUB deactivate the kmem_caches through call_rcu. Make

	 * sure all registered rcu callbacks have been invoked.

	 * sure all registered rcu callbacks have been invoked.

	 */

	 */

	if (IS_ENABLED(CONFIG_SLUB))

	rcu_barrier();

	rcu_barrier();





	/*

	/*

mm/slub.c

+1 −7
Original line number Original line Diff line number Diff line
@@ -4008,7 +4008,7 @@ int __kmem_cache_shrink(struct kmem_cache *s) 
}

}





#ifdef CONFIG_MEMCG

#ifdef CONFIG_MEMCG

static void kmemcg_cache_deact_after_rcu(struct kmem_cache *s)

void __kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s)

{

{

	/*

	/*

	 * Called with all the locks held after a sched RCU grace period.

	 * Called with all the locks held after a sched RCU grace period.
@@ -4034,12 +4034,6 @@ void __kmemcg_cache_deactivate(struct kmem_cache *s) 
	 */

	 */

	slub_set_cpu_partial(s, 0);

	slub_set_cpu_partial(s, 0);

	s->min_partial = 0;

	s->min_partial = 0;



	/*

	 * s->cpu_partial is checked locklessly (see put_cpu_partial), so

	 * we have to make sure the change is visible before shrinking.

	 */

	slab_deactivate_memcg_cache_rcu_sched(s, kmemcg_cache_deact_after_rcu);

}

}

#endif	/* CONFIG_MEMCG */

#endif	/* CONFIG_MEMCG */

CRA Git | Maintained and supported by SUSTech CRA and CCSE