Commit c0ff7453 authored by Miao Xie's avatar Miao Xie Committed by Linus Torvalds
Browse files

cpuset,mm: fix no node to alloc memory when changing cpuset's mems 



Before applying this patch, cpuset updates task->mems_allowed and
mempolicy by setting all new bits in the nodemask first, and clearing all
old unallowed bits later.  But in the way, the allocator may find that
there is no node to alloc memory.

The reason is that cpuset rebinds the task's mempolicy, it cleans the
nodes which the allocater can alloc pages on, for example:

(mpol: mempolicy)
	task1			task1's mpol	task2
	alloc page		1
	  alloc on node0? NO	1
				1		change mems from 1 to 0
				1		rebind task1's mpol
				0-1		  set new bits
				0	  	  clear disallowed bits
	  alloc on node1? NO	0
	  ...
	can't alloc page
	  goto oom

This patch fixes this problem by expanding the nodes range first(set newly
allowed bits) and shrink it lazily(clear newly disallowed bits).  So we
use a variable to tell the write-side task that read-side task is reading
nodemask, and the write-side task clears newly disallowed nodes after
read-side task ends the current memory allocation.

[akpm@linux-foundation.org: fix spello]
Signed-off-by: default avatarMiao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Paul Menage <menage@google.com>
Cc: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Ravikiran Thirumalai <kiran@scalex86.org>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 708c1bbc

include/linux/cpuset.h

+43 −0
Original line number Diff line number Diff line
@@ -86,9 +86,44 @@ extern void rebuild_sched_domains(void); 


extern void cpuset_print_task_mems_allowed(struct task_struct *p);



/*

 * reading current mems_allowed and mempolicy in the fastpath must protected

 * by get_mems_allowed()

 */

static inline void get_mems_allowed(void)

{

	current->mems_allowed_change_disable++;



	/*

	 * ensure that reading mems_allowed and mempolicy happens after the

	 * update of ->mems_allowed_change_disable.

	 *

	 * the write-side task finds ->mems_allowed_change_disable is not 0,

	 * and knows the read-side task is reading mems_allowed or mempolicy,

	 * so it will clear old bits lazily.

	 */

	smp_mb();

}



static inline void put_mems_allowed(void)

{

	/*

	 * ensure that reading mems_allowed and mempolicy before reducing

	 * mems_allowed_change_disable.

	 *

	 * the write-side task will know that the read-side task is still

	 * reading mems_allowed or mempolicy, don't clears old bits in the

	 * nodemask.

	 */

	smp_mb();

	--ACCESS_ONCE(current->mems_allowed_change_disable);

}



static inline void set_mems_allowed(nodemask_t nodemask)

{

	task_lock(current);

	current->mems_allowed = nodemask;

	task_unlock(current);

}



#else /* !CONFIG_CPUSETS */
@@ -187,6 +222,14 @@ static inline void set_mems_allowed(nodemask_t nodemask) 
{

}



static inline void get_mems_allowed(void)

{

}



static inline void put_mems_allowed(void)

{

}



#endif /* !CONFIG_CPUSETS */



#endif /* _LINUX_CPUSET_H */

include/linux/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -1421,6 +1421,7 @@ struct task_struct { 
#endif

#ifdef CONFIG_CPUSETS

	nodemask_t mems_allowed;	/* Protected by alloc_lock */

	int mems_allowed_change_disable;

	int cpuset_mem_spread_rotor;

#endif

#ifdef CONFIG_CGROUPS

kernel/cpuset.c

+50 −8
Original line number Diff line number Diff line
@@ -946,16 +946,62 @@ static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from, 
 * In order to avoid seeing no nodes if the old and new nodes are disjoint,

 * we structure updates as setting all new allowed nodes, then clearing newly

 * disallowed ones.

 *

 * Called with task's alloc_lock held

 */

static void cpuset_change_task_nodemask(struct task_struct *tsk,

					nodemask_t *newmems)

{

repeat:

	/*

	 * Allow tasks that have access to memory reserves because they have

	 * been OOM killed to get memory anywhere.

	 */

	if (unlikely(test_thread_flag(TIF_MEMDIE)))

		return;

	if (current->flags & PF_EXITING) /* Let dying task have memory */

		return;



	task_lock(tsk);

	nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);

	mpol_rebind_task(tsk, &tsk->mems_allowed, MPOL_REBIND_ONCE);

	mpol_rebind_task(tsk, newmems, MPOL_REBIND_ONCE);

	mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1);





	/*

	 * ensure checking ->mems_allowed_change_disable after setting all new

	 * allowed nodes.

	 *

	 * the read-side task can see an nodemask with new allowed nodes and

	 * old allowed nodes. and if it allocates page when cpuset clears newly

	 * disallowed ones continuous, it can see the new allowed bits.

	 *

	 * And if setting all new allowed nodes is after the checking, setting

	 * all new allowed nodes and clearing newly disallowed ones will be done

	 * continuous, and the read-side task may find no node to alloc page.

	 */

	smp_mb();



	/*

	 * Allocation of memory is very fast, we needn't sleep when waiting

	 * for the read-side.

	 */

	while (ACCESS_ONCE(tsk->mems_allowed_change_disable)) {

		task_unlock(tsk);

		if (!task_curr(tsk))

			yield();

		goto repeat;

	}



	/*

	 * ensure checking ->mems_allowed_change_disable before clearing all new

	 * disallowed nodes.

	 *

	 * if clearing newly disallowed bits before the checking, the read-side

	 * task may find no node to alloc page.

	 */

	smp_mb();



	mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2);

	tsk->mems_allowed = *newmems;

	task_unlock(tsk);

}



/*
@@ -978,9 +1024,7 @@ static void cpuset_change_nodemask(struct task_struct *p, 
	cs = cgroup_cs(scan->cg);

	guarantee_online_mems(cs, newmems);



	task_lock(p);

	cpuset_change_task_nodemask(p, newmems);

	task_unlock(p);



	NODEMASK_FREE(newmems);
@@ -1383,9 +1427,7 @@ static void cpuset_attach_task(struct task_struct *tsk, nodemask_t *to, 
	err = set_cpus_allowed_ptr(tsk, cpus_attach);

	WARN_ON_ONCE(err);



	task_lock(tsk);

	cpuset_change_task_nodemask(tsk, to);

	task_unlock(tsk);

	cpuset_update_task_spread_flag(cs, tsk);



}

kernel/exit.c

+2 −0
Original line number Diff line number Diff line
@@ -1002,8 +1002,10 @@ NORET_TYPE void do_exit(long code) 


	exit_notify(tsk, group_dead);

#ifdef CONFIG_NUMA

	task_lock(tsk);

	mpol_put(tsk->mempolicy);

	tsk->mempolicy = NULL;

	task_unlock(tsk);

#endif

#ifdef CONFIG_FUTEX

	if (unlikely(current->pi_state_cache))

mm/filemap.c

+8 −2
Original line number Diff line number Diff line
@@ -461,9 +461,15 @@ EXPORT_SYMBOL_GPL(add_to_page_cache_lru); 
#ifdef CONFIG_NUMA

struct page *__page_cache_alloc(gfp_t gfp)

{

	int n;

	struct page *page;



	if (cpuset_do_page_mem_spread()) {

		int n = cpuset_mem_spread_node();

		return alloc_pages_exact_node(n, gfp, 0);

		get_mems_allowed();

		n = cpuset_mem_spread_node();

		page = alloc_pages_exact_node(n, gfp, 0);

		put_mems_allowed();

		return page;

	}

	return alloc_pages(gfp, 0);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE