Commit 3dfa5721 authored by Christoph Lameter's avatar Christoph Lameter Committed by Linus Torvalds
Browse files

Page allocator: get rid of the list of cold pages 

We have repeatedly discussed if the cold pages still have a point. There is
one way to join the two lists: Use a single list and put the cold pages at the
end and the hot pages at the beginning. That way a single list can serve for
both types of allocations.

The discussion of the RFC for this and Mel's measurements indicate that
there may not be too much of a point left to having separate lists for
hot and cold pages (see http://marc.info/?t=119492914200001&r=1&w=2

).

Signed-off-by: default avatarChristoph Lameter <clameter@sgi.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Martin Bligh <mbligh@mbligh.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5dc33185

include/linux/mmzone.h

+1 −1
Original line number Diff line number Diff line
@@ -113,7 +113,7 @@ struct per_cpu_pages { 
};



struct per_cpu_pageset {

	struct per_cpu_pages pcp[2];	/* 0: hot.  1: cold */

	struct per_cpu_pages pcp;

#ifdef CONFIG_NUMA

	s8 expire;

#endif

mm/page_alloc.c

+27 −30
Original line number Diff line number Diff line
@@ -901,26 +901,23 @@ static void drain_pages(unsigned int cpu) 
{

	unsigned long flags;

	struct zone *zone;

	int i;



	for_each_zone(zone) {

		struct per_cpu_pageset *pset;

		struct per_cpu_pages *pcp;



		if (!populated_zone(zone))

			continue;



		pset = zone_pcp(zone, cpu);

		for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) {

			struct per_cpu_pages *pcp;



			pcp = &pset->pcp[i];

		pcp = &pset->pcp;

		local_irq_save(flags);

		free_pages_bulk(zone, pcp->count, &pcp->list, 0);

		pcp->count = 0;

		local_irq_restore(flags);

	}

}

}



/*

 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
@@ -993,9 +990,12 @@ static void fastcall free_hot_cold_page(struct page *page, int cold) 
	arch_free_page(page, 0);

	kernel_map_pages(page, 1, 0);



	pcp = &zone_pcp(zone, get_cpu())->pcp[cold];

	pcp = &zone_pcp(zone, get_cpu())->pcp;

	local_irq_save(flags);

	__count_vm_event(PGFREE);

	if (cold)

		list_add_tail(&page->lru, &pcp->list);

	else

		list_add(&page->lru, &pcp->list);

	set_page_private(page, get_pageblock_migratetype(page));

	pcp->count++;
@@ -1054,7 +1054,7 @@ again: 
	if (likely(order == 0)) {

		struct per_cpu_pages *pcp;



		pcp = &zone_pcp(zone, cpu)->pcp[cold];

		pcp = &zone_pcp(zone, cpu)->pcp;

		local_irq_save(flags);

		if (!pcp->count) {

			pcp->count = rmqueue_bulk(zone, 0,
@@ -1064,9 +1064,15 @@ again: 
		}



		/* Find a page of the appropriate migrate type */

		if (cold) {

			list_for_each_entry_reverse(page, &pcp->list, lru)

				if (page_private(page) == migratetype)

					break;

		} else {

			list_for_each_entry(page, &pcp->list, lru)

				if (page_private(page) == migratetype)

					break;

		}



		/* Allocate more to the pcp list if necessary */

		if (unlikely(&page->lru == &pcp->list)) {
@@ -1793,12 +1799,9 @@ void show_free_areas(void) 


			pageset = zone_pcp(zone, cpu);



			printk("CPU %4d: Hot: hi:%5d, btch:%4d usd:%4d   "

			       "Cold: hi:%5d, btch:%4d usd:%4d\n",

			       cpu, pageset->pcp[0].high,

			       pageset->pcp[0].batch, pageset->pcp[0].count,

			       pageset->pcp[1].high, pageset->pcp[1].batch,

			       pageset->pcp[1].count);

			printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",

			       cpu, pageset->pcp.high,

			       pageset->pcp.batch, pageset->pcp.count);

		}

	}
@@ -2596,17 +2599,11 @@ inline void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 


	memset(p, 0, sizeof(*p));



	pcp = &p->pcp[0];		/* hot */

	pcp = &p->pcp;

	pcp->count = 0;

	pcp->high = 6 * batch;

	pcp->batch = max(1UL, 1 * batch);

	INIT_LIST_HEAD(&pcp->list);



	pcp = &p->pcp[1];		/* cold*/

	pcp->count = 0;

	pcp->high = 2 * batch;

	pcp->batch = max(1UL, batch/2);

	INIT_LIST_HEAD(&pcp->list);

}



/*
@@ -2619,7 +2616,7 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p, 
{

	struct per_cpu_pages *pcp;



	pcp = &p->pcp[0]; /* hot list */

	pcp = &p->pcp;

	pcp->high = high;

	pcp->batch = max(1UL, high/4);

	if ((high/4) > (PAGE_SHIFT * 8))

mm/vmstat.c

+12 −18
Original line number Diff line number Diff line
@@ -337,7 +337,7 @@ void refresh_cpu_vm_stats(int cpu) 
		 * Check if there are pages remaining in this pageset

		 * if not then there is nothing to expire.

		 */

		if (!p->expire || (!p->pcp[0].count && !p->pcp[1].count))

		if (!p->expire || !p->pcp.count)

			continue;



		/*
@@ -352,11 +352,8 @@ void refresh_cpu_vm_stats(int cpu) 
		if (p->expire)

			continue;



		if (p->pcp[0].count)

			drain_zone_pages(zone, p->pcp + 0);



		if (p->pcp[1].count)

			drain_zone_pages(zone, p->pcp + 1);

		if (p->pcp.count)

			drain_zone_pages(zone, &p->pcp);

#endif

	}
@@ -693,20 +690,17 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 
		   "\n  pagesets");

	for_each_online_cpu(i) {

		struct per_cpu_pageset *pageset;

		int j;



		pageset = zone_pcp(zone, i);

		for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {

		seq_printf(m,

				   "\n    cpu: %i pcp: %i"

			   "\n    cpu: %i"

			   "\n              count: %i"

			   "\n              high:  %i"

			   "\n              batch: %i",

				   i, j,

				   pageset->pcp[j].count,

				   pageset->pcp[j].high,

				   pageset->pcp[j].batch);

			}

			   i,

			   pageset->pcp.count,

			   pageset->pcp.high,

			   pageset->pcp.batch);

#ifdef CONFIG_SMP

		seq_printf(m, "\n  vm stats threshold: %d",

				pageset->stat_threshold);

CRA Git | Maintained and supported by SUSTech CRA and CCSE