Commit 854ffa8d authored by Daisuke Nishimura's avatar Daisuke Nishimura Committed by Linus Torvalds
Browse files

memcg: improve performance in moving charge 



Try to reduce overheads in moving charge by:

- Instead of calling res_counter_uncharge() against the old cgroup in
  __mem_cgroup_move_account() everytime, call res_counter_uncharge() at the end
  of task migration once.
- removed css_get(&to->css) from __mem_cgroup_move_account() because callers
  should have already called css_get(). And removed css_put(&to->css) too,
  which was called by callers of move_account on success of move_account.
- Instead of calling __mem_cgroup_try_charge(), i.e. res_counter_charge(),
  repeatedly, call res_counter_charge(PAGE_SIZE * count) in can_attach() if
  possible.
- Instead of calling css_get()/css_put() repeatedly, make use of coalesce
  __css_get()/__css_put() if possible.

These changes reduces the overhead from 1.7sec to 0.6sec to move charges
of 1G anonymous memory in my test environment.

Signed-off-by: default avatarDaisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Paul Menage <menage@google.com>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4ffef5fe

mm/memcontrol.c

+98 −54
Original line number Diff line number Diff line
@@ -253,6 +253,7 @@ static struct move_charge_struct { 
	struct mem_cgroup *from;

	struct mem_cgroup *to;

	unsigned long precharge;

	unsigned long moved_charge;

} mc;



/*
@@ -1536,14 +1537,23 @@ nomem: 
 * This function is for that and do uncharge, put css's refcnt.

 * gotten by try_charge().

 */

static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)

static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,

							unsigned long count)

{

	if (!mem_cgroup_is_root(mem)) {

		res_counter_uncharge(&mem->res, PAGE_SIZE);

		res_counter_uncharge(&mem->res, PAGE_SIZE * count);

		if (do_swap_account)

			res_counter_uncharge(&mem->memsw, PAGE_SIZE);

			res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);

		VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));

		WARN_ON_ONCE(count > INT_MAX);

		__css_put(&mem->css, (int)count);

	}

	css_put(&mem->css);

	/* we don't need css_put for root */

}



static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)

{

	__mem_cgroup_cancel_charge(mem, 1);

}



/*
@@ -1646,17 +1656,20 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, 
 * @pc:	page_cgroup of the page.

 * @from: mem_cgroup which the page is moved from.

 * @to:	mem_cgroup which the page is moved to. @from != @to.

 * @uncharge: whether we should call uncharge and css_put against @from.

 *

 * The caller must confirm following.

 * - page is not on LRU (isolate_page() is useful.)

 * - the pc is locked, used, and ->mem_cgroup points to @from.

 *

 * This function does "uncharge" from old cgroup but doesn't do "charge" to

 * new cgroup. It should be done by a caller.

 * This function doesn't do "charge" nor css_get to new cgroup. It should be

 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is

 * true, this function does "uncharge" from old cgroup, but it doesn't if

 * @uncharge is false, so a caller should do "uncharge".

 */



static void __mem_cgroup_move_account(struct page_cgroup *pc,

	struct mem_cgroup *from, struct mem_cgroup *to)

	struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)

{

	struct page *page;

	int cpu;
@@ -1669,10 +1682,6 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc, 
	VM_BUG_ON(!PageCgroupUsed(pc));

	VM_BUG_ON(pc->mem_cgroup != from);



	if (!mem_cgroup_is_root(from))

		res_counter_uncharge(&from->res, PAGE_SIZE);

	mem_cgroup_charge_statistics(from, pc, false);



	page = pc->page;

	if (page_mapped(page) && !PageAnon(page)) {

		cpu = smp_processor_id();
@@ -1688,12 +1697,12 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc, 
		__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_FILE_MAPPED,

						1);

	}

	mem_cgroup_charge_statistics(from, pc, false);

	if (uncharge)

		/* This is not "cancel", but cancel_charge does all we need. */

		mem_cgroup_cancel_charge(from);



	if (do_swap_account && !mem_cgroup_is_root(from))

		res_counter_uncharge(&from->memsw, PAGE_SIZE);

	css_put(&from->css);



	css_get(&to->css);

	/* caller should have done css_get */

	pc->mem_cgroup = to;

	mem_cgroup_charge_statistics(to, pc, true);

	/*
@@ -1710,12 +1719,12 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc, 
 * __mem_cgroup_move_account()

 */

static int mem_cgroup_move_account(struct page_cgroup *pc,

				struct mem_cgroup *from, struct mem_cgroup *to)

		struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)

{

	int ret = -EINVAL;

	lock_page_cgroup(pc);

	if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {

		__mem_cgroup_move_account(pc, from, to);

		__mem_cgroup_move_account(pc, from, to, uncharge);

		ret = 0;

	}

	unlock_page_cgroup(pc);
@@ -1751,11 +1760,9 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc, 
	if (ret || !parent)

		goto put_back;



	ret = mem_cgroup_move_account(pc, child, parent);

	if (!ret)

		css_put(&parent->css);	/* drop extra refcnt by try_charge() */

	else

		mem_cgroup_cancel_charge(parent);	/* does css_put */

	ret = mem_cgroup_move_account(pc, child, parent, true);

	if (ret)

		mem_cgroup_cancel_charge(parent);

put_back:

	putback_lru_page(page);

put:
@@ -3438,16 +3445,58 @@ static int mem_cgroup_populate(struct cgroup_subsys *ss, 
}



/* Handlers for move charge at task migration. */

static int mem_cgroup_do_precharge(void)

#define PRECHARGE_COUNT_AT_ONCE	256

static int mem_cgroup_do_precharge(unsigned long count)

{

	int ret = -ENOMEM;

	int ret = 0;

	int batch_count = PRECHARGE_COUNT_AT_ONCE;

	struct mem_cgroup *mem = mc.to;



	ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false, NULL);

	if (mem_cgroup_is_root(mem)) {

		mc.precharge += count;

		/* we don't need css_get for root */

		return ret;

	}

	/* try to charge at once */

	if (count > 1) {

		struct res_counter *dummy;

		/*

		 * "mem" cannot be under rmdir() because we've already checked

		 * by cgroup_lock_live_cgroup() that it is not removed and we

		 * are still under the same cgroup_mutex. So we can postpone

		 * css_get().

		 */

		if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))

			goto one_by_one;

		if (do_swap_account && res_counter_charge(&mem->memsw,

						PAGE_SIZE * count, &dummy)) {

			res_counter_uncharge(&mem->res, PAGE_SIZE * count);

			goto one_by_one;

		}

		mc.precharge += count;

		VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));

		WARN_ON_ONCE(count > INT_MAX);

		__css_get(&mem->css, (int)count);

		return ret;

	}

one_by_one:

	/* fall back to one by one charge */

	while (count--) {

		if (signal_pending(current)) {

			ret = -EINTR;

			break;

		}

		if (!batch_count--) {

			batch_count = PRECHARGE_COUNT_AT_ONCE;

			cond_resched();

		}

		ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem,

								false, NULL);

		if (ret || !mem)

			/* mem_cgroup_clear_mc() will do uncharge later */

			return -ENOMEM;



		mc.precharge++;

	}

	return ret;

}
@@ -3570,34 +3619,25 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 
	return precharge;

}



#define PRECHARGE_AT_ONCE	256

static int mem_cgroup_precharge_mc(struct mm_struct *mm)

{

	int ret = 0;

	int count = PRECHARGE_AT_ONCE;

	unsigned long precharge = mem_cgroup_count_precharge(mm);



	while (!ret && precharge--) {

		if (signal_pending(current)) {

			ret = -EINTR;

			break;

		}

		if (!count--) {

			count = PRECHARGE_AT_ONCE;

			cond_resched();

		}

		ret = mem_cgroup_do_precharge();

	}



	return ret;

	return mem_cgroup_do_precharge(mem_cgroup_count_precharge(mm));

}



static void mem_cgroup_clear_mc(void)

{

	/* we must uncharge all the leftover precharges from mc.to */

	while (mc.precharge) {

		mem_cgroup_cancel_charge(mc.to);

		mc.precharge--;

	if (mc.precharge) {

		__mem_cgroup_cancel_charge(mc.to, mc.precharge);

		mc.precharge = 0;

	}

	/*

	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so

	 * we must uncharge here.

	 */

	if (mc.moved_charge) {

		__mem_cgroup_cancel_charge(mc.from, mc.moved_charge);

		mc.moved_charge = 0;

	}

	mc.from = NULL;

	mc.to = NULL;
@@ -3625,9 +3665,11 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 
			VM_BUG_ON(mc.from);

			VM_BUG_ON(mc.to);

			VM_BUG_ON(mc.precharge);

			VM_BUG_ON(mc.moved_charge);

			mc.from = from;

			mc.to = mem;

			mc.precharge = 0;

			mc.moved_charge = 0;



			ret = mem_cgroup_precharge_mc(mm);

			if (ret)
@@ -3674,9 +3716,11 @@ retry: 
			if (isolate_lru_page(page))

				goto put;

			pc = lookup_page_cgroup(page);

			if (!mem_cgroup_move_account(pc, mc.from, mc.to)) {

				css_put(&mc.to->css);

			if (!mem_cgroup_move_account(pc,

						mc.from, mc.to, false)) {

				mc.precharge--;

				/* we uncharge from mc.from later. */

				mc.moved_charge++;

			}

			putback_lru_page(page);

put:			/* is_target_pte_for_mc() gets the page */
@@ -3696,7 +3740,7 @@ put: /* is_target_pte_for_mc() gets the page */ 
		 * charges to mc.to if we have failed in charge once in attach()

		 * phase.

		 */

		ret = mem_cgroup_do_precharge();

		ret = mem_cgroup_do_precharge(1);

		if (!ret)

			goto retry;

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE