Commit 93e4a89a authored by KOSAKI Motohiro's avatar KOSAKI Motohiro Committed by Linus Torvalds
Browse files

mm: restore zone->all_unreclaimable to independence word 



commit e815af95 ("change all_unreclaimable zone member to flags") changed
all_unreclaimable member to bit flag.  But it had an undesireble side
effect.  free_one_page() is one of most hot path in linux kernel and
increasing atomic ops in it can reduce kernel performance a bit.

Thus, this patch revert such commit partially. at least
all_unreclaimable shouldn't share memory word with other zone flags.

[akpm@linux-foundation.org: fix patch interaction]
Signed-off-by: default avatarKOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Huang Shijie <shijie8@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent fc91668e

include/linux/mmzone.h

+1 −6
Original line number Original line Diff line number Diff line
@@ -306,6 +306,7 @@ struct zone { 
	 * free areas of different sizes

	 * free areas of different sizes

	 */

	 */

	spinlock_t		lock;

	spinlock_t		lock;

	int                     all_unreclaimable; /* All pages pinned */

#ifdef CONFIG_MEMORY_HOTPLUG

#ifdef CONFIG_MEMORY_HOTPLUG

	/* see spanned/present_pages for more description */

	/* see spanned/present_pages for more description */

	seqlock_t		span_seqlock;

	seqlock_t		span_seqlock;
@@ -417,7 +418,6 @@ struct zone { 
} ____cacheline_internodealigned_in_smp;

} ____cacheline_internodealigned_in_smp;





typedef enum {

typedef enum {

	ZONE_ALL_UNRECLAIMABLE,		/* all pages pinned */

	ZONE_RECLAIM_LOCKED,		/* prevents concurrent reclaim */

	ZONE_RECLAIM_LOCKED,		/* prevents concurrent reclaim */

	ZONE_OOM_LOCKED,		/* zone is in OOM killer zonelist */

	ZONE_OOM_LOCKED,		/* zone is in OOM killer zonelist */

} zone_flags_t;

} zone_flags_t;
@@ -437,11 +437,6 @@ static inline void zone_clear_flag(struct zone *zone, zone_flags_t flag) 
	clear_bit(flag, &zone->flags);

	clear_bit(flag, &zone->flags);

}

}





static inline int zone_is_all_unreclaimable(const struct zone *zone)

{

	return test_bit(ZONE_ALL_UNRECLAIMABLE, &zone->flags);

}



static inline int zone_is_reclaim_locked(const struct zone *zone)

static inline int zone_is_reclaim_locked(const struct zone *zone)

{

{

	return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags);

	return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags);

mm/page_alloc.c

+3 −3
Original line number Original line Diff line number Diff line
@@ -530,7 +530,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, 
	int batch_free = 0;

	int batch_free = 0;





	spin_lock(&zone->lock);

	spin_lock(&zone->lock);

	zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);

	zone->all_unreclaimable = 0;

	zone->pages_scanned = 0;

	zone->pages_scanned = 0;





	__mod_zone_page_state(zone, NR_FREE_PAGES, count);

	__mod_zone_page_state(zone, NR_FREE_PAGES, count);
@@ -568,7 +568,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order, 
				int migratetype)

				int migratetype)

{

{

	spin_lock(&zone->lock);

	spin_lock(&zone->lock);

	zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);

	zone->all_unreclaimable = 0;

	zone->pages_scanned = 0;

	zone->pages_scanned = 0;





	__mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);

	__mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
@@ -2262,7 +2262,7 @@ void show_free_areas(void) 
			K(zone_page_state(zone, NR_BOUNCE)),

			K(zone_page_state(zone, NR_BOUNCE)),

			K(zone_page_state(zone, NR_WRITEBACK_TEMP)),

			K(zone_page_state(zone, NR_WRITEBACK_TEMP)),

			zone->pages_scanned,

			zone->pages_scanned,

			(zone_is_all_unreclaimable(zone) ? "yes" : "no")

			(zone->all_unreclaimable ? "yes" : "no")

			);

			);

		printk("lowmem_reserve[]:");

		printk("lowmem_reserve[]:");

		for (i = 0; i < MAX_NR_ZONES; i++)

		for (i = 0; i < MAX_NR_ZONES; i++)

mm/vmscan.c

+9 −13
Original line number Original line Diff line number Diff line
@@ -1699,8 +1699,7 @@ static void shrink_zones(int priority, struct zonelist *zonelist, 
				continue;

				continue;

			note_zone_scanning_priority(zone, priority);

			note_zone_scanning_priority(zone, priority);





			if (zone_is_all_unreclaimable(zone) &&

			if (zone->all_unreclaimable && priority != DEF_PRIORITY)

						priority != DEF_PRIORITY)

				continue;	/* Let kswapd poll it */

				continue;	/* Let kswapd poll it */

			sc->all_unreclaimable = 0;

			sc->all_unreclaimable = 0;

		} else {

		} else {
@@ -1927,7 +1926,7 @@ static int sleeping_prematurely(pg_data_t *pgdat, int order, long remaining) 
		if (!populated_zone(zone))

		if (!populated_zone(zone))

			continue;

			continue;





		if (zone_is_all_unreclaimable(zone))

		if (zone->all_unreclaimable)

			continue;

			continue;





		if (!zone_watermark_ok(zone, order, high_wmark_pages(zone),

		if (!zone_watermark_ok(zone, order, high_wmark_pages(zone),
@@ -2017,8 +2016,7 @@ loop_again: 
			if (!populated_zone(zone))

			if (!populated_zone(zone))

				continue;

				continue;





			if (zone_is_all_unreclaimable(zone) &&

			if (zone->all_unreclaimable && priority != DEF_PRIORITY)

			    priority != DEF_PRIORITY)

				continue;

				continue;





			/*

			/*
@@ -2061,8 +2059,7 @@ loop_again: 
			if (!populated_zone(zone))

			if (!populated_zone(zone))

				continue;

				continue;





			if (zone_is_all_unreclaimable(zone) &&

			if (zone->all_unreclaimable && priority != DEF_PRIORITY)

					priority != DEF_PRIORITY)

				continue;

				continue;





			temp_priority[i] = priority;

			temp_priority[i] = priority;
@@ -2089,12 +2086,11 @@ loop_again: 
						lru_pages);

						lru_pages);

			sc.nr_reclaimed += reclaim_state->reclaimed_slab;

			sc.nr_reclaimed += reclaim_state->reclaimed_slab;

			total_scanned += sc.nr_scanned;

			total_scanned += sc.nr_scanned;

			if (zone_is_all_unreclaimable(zone))

			if (zone->all_unreclaimable)

				continue;

				continue;

			if (nr_slab == 0 && zone->pages_scanned >=

			if (nr_slab == 0 &&

					(zone_reclaimable_pages(zone) * 6))

			    zone->pages_scanned >= (zone_reclaimable_pages(zone) * 6))

					zone_set_flag(zone,

				zone->all_unreclaimable = 1;

						      ZONE_ALL_UNRECLAIMABLE);

			/*

			/*

			 * If we've done a decent amount of scanning and

			 * If we've done a decent amount of scanning and

			 * the reclaim ratio is low, start doing writepage

			 * the reclaim ratio is low, start doing writepage
@@ -2624,7 +2620,7 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) 
	    zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)

	    zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)

		return ZONE_RECLAIM_FULL;

		return ZONE_RECLAIM_FULL;





	if (zone_is_all_unreclaimable(zone))

	if (zone->all_unreclaimable)

		return ZONE_RECLAIM_FULL;

		return ZONE_RECLAIM_FULL;





	/*

	/*

mm/vmstat.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -763,7 +763,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 
		   "\n  prev_priority:     %i"

		   "\n  prev_priority:     %i"

		   "\n  start_pfn:         %lu"

		   "\n  start_pfn:         %lu"

		   "\n  inactive_ratio:    %u",

		   "\n  inactive_ratio:    %u",

			   zone_is_all_unreclaimable(zone),

		   zone->all_unreclaimable,

		   zone->prev_priority,

		   zone->prev_priority,

		   zone->zone_start_pfn,

		   zone->zone_start_pfn,

		   zone->inactive_ratio);

		   zone->inactive_ratio);

CRA Git | Maintained and supported by SUSTech CRA and CCSE