Commit 52383431 authored by Vladimir Davydov's avatar Vladimir Davydov Committed by Linus Torvalds
Browse files

mm: get rid of __GFP_KMEMCG 



Currently to allocate a page that should be charged to kmemcg (e.g.
threadinfo), we pass __GFP_KMEMCG flag to the page allocator.  The page
allocated is then to be freed by free_memcg_kmem_pages.  Apart from
looking asymmetrical, this also requires intrusion to the general
allocation path.  So let's introduce separate functions that will
alloc/free pages charged to kmemcg.

The new functions are called alloc_kmem_pages and free_kmem_pages.  They
should be used when the caller actually would like to use kmalloc, but
has to fall back to the page allocator for the allocation is large.
They only differ from alloc_pages and free_pages in that besides
allocating or freeing pages they also charge them to the kmem resource
counter of the current memory cgroup.

[sfr@canb.auug.org.au: export kmalloc_order() to modules]
Signed-off-by: default avatarVladimir Davydov <vdavydov@parallels.com>
Acked-by: default avatarGreg Thelen <gthelen@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarMichal Hocko <mhocko@suse.cz>
Cc: Glauber Costa <glommer@gmail.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Pekka Enberg <penberg@kernel.org>
Signed-off-by: default avatarStephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5dfb4175

include/linux/gfp.h

+6 −4
Original line number Diff line number Diff line
@@ -31,7 +31,6 @@ struct vm_area_struct; 
#define ___GFP_HARDWALL		0x20000u

#define ___GFP_THISNODE		0x40000u

#define ___GFP_RECLAIMABLE	0x80000u

#define ___GFP_KMEMCG		0x100000u

#define ___GFP_NOTRACK		0x200000u

#define ___GFP_NO_KSWAPD	0x400000u

#define ___GFP_OTHER_NODE	0x800000u
@@ -91,7 +90,6 @@ struct vm_area_struct; 


#define __GFP_NO_KSWAPD	((__force gfp_t)___GFP_NO_KSWAPD)

#define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */

#define __GFP_KMEMCG	((__force gfp_t)___GFP_KMEMCG) /* Allocation comes from a memcg-accounted resource */

#define __GFP_WRITE	((__force gfp_t)___GFP_WRITE)	/* Allocator intends to dirty page */



/*
@@ -353,6 +351,10 @@ extern struct page *alloc_pages_vma(gfp_t gfp_mask, int order, 
#define alloc_page_vma_node(gfp_mask, vma, addr, node)		\

	alloc_pages_vma(gfp_mask, 0, vma, addr, node)



extern struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order);

extern struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask,

					  unsigned int order);



extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order);

extern unsigned long get_zeroed_page(gfp_t gfp_mask);
@@ -372,8 +374,8 @@ extern void free_pages(unsigned long addr, unsigned int order); 
extern void free_hot_cold_page(struct page *page, int cold);

extern void free_hot_cold_page_list(struct list_head *list, int cold);



extern void __free_memcg_kmem_pages(struct page *page, unsigned int order);

extern void free_memcg_kmem_pages(unsigned long addr, unsigned int order);

extern void __free_kmem_pages(struct page *page, unsigned int order);

extern void free_kmem_pages(unsigned long addr, unsigned int order);



#define __free_page(page) __free_pages((page), 0)

#define free_page(addr) free_pages((addr), 0)

include/linux/memcontrol.h

+1 −1
Original line number Diff line number Diff line
@@ -537,7 +537,7 @@ memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order) 
	 * res_counter_charge_nofail, but we hope those allocations are rare,

	 * and won't be worth the trouble.

	 */

	if (!(gfp & __GFP_KMEMCG) || (gfp & __GFP_NOFAIL))

	if (gfp & __GFP_NOFAIL)

		return true;

	if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD))

		return true;

include/linux/slab.h

+1 −10
Original line number Diff line number Diff line
@@ -369,16 +369,7 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, 
#include <linux/slub_def.h>

#endif



static __always_inline void *

kmalloc_order(size_t size, gfp_t flags, unsigned int order)

{

	void *ret;



	flags |= (__GFP_COMP | __GFP_KMEMCG);

	ret = (void *) __get_free_pages(flags, order);

	kmemleak_alloc(ret, size, 1, flags);

	return ret;

}

extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order);



#ifdef CONFIG_TRACING

extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);

include/linux/thread_info.h

+0 −2
Original line number Diff line number Diff line
@@ -61,8 +61,6 @@ extern long do_no_restart_syscall(struct restart_block *parm); 
# define THREADINFO_GFP		(GFP_KERNEL | __GFP_NOTRACK)

#endif



#define THREADINFO_GFP_ACCOUNTED (THREADINFO_GFP | __GFP_KMEMCG)



/*

 * flag set/clear/test wrappers

 * - pass TIF_xxxx constants to these functions

include/trace/events/gfpflags.h

+0 −1
Original line number Diff line number Diff line
@@ -34,7 +34,6 @@ 
	{(unsigned long)__GFP_HARDWALL,		"GFP_HARDWALL"},	\

	{(unsigned long)__GFP_THISNODE,		"GFP_THISNODE"},	\

	{(unsigned long)__GFP_RECLAIMABLE,	"GFP_RECLAIMABLE"},	\

	{(unsigned long)__GFP_KMEMCG,		"GFP_KMEMCG"},		\

	{(unsigned long)__GFP_MOVABLE,		"GFP_MOVABLE"},		\

	{(unsigned long)__GFP_NOTRACK,		"GFP_NOTRACK"},		\

	{(unsigned long)__GFP_NO_KSWAPD,	"GFP_NO_KSWAPD"},	\

CRA Git | Maintained and supported by SUSTech CRA and CCSE