Commit dd1a239f authored by Mel Gorman's avatar Mel Gorman Committed by Linus Torvalds
Browse files

mm: have zonelist contains structs with both a zone pointer and zone_idx 



Filtering zonelists requires very frequent use of zone_idx().  This is costly
as it involves a lookup of another structure and a substraction operation.  As
the zone_idx is often required, it should be quickly accessible.  The node idx
could also be stored here if it was found that accessing zone->node is
significant which may be the case on workloads where nodemasks are heavily
used.

This patch introduces a struct zoneref to store a zone pointer and a zone
index.  The zonelist then consists of an array of these struct zonerefs which
are looked up as necessary.  Helpers are given for accessing the zone index as
well as the node index.

[kamezawa.hiroyu@jp.fujitsu.com: Suggested struct zoneref instead of embedding information in pointers]
[hugh@veritas.com: mm-have-zonelist: fix memcg ooms]
[hugh@veritas.com: just return do_try_to_free_pages]
[hugh@veritas.com: do_try_to_free_pages gfp_mask redundant]
Signed-off-by: default avatarMel Gorman <mel@csn.ul.ie>
Acked-by: default avatarChristoph Lameter <clameter@sgi.com>
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Signed-off-by: default avatarLee Schermerhorn <lee.schermerhorn@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: default avatarHugh Dickins <hugh@veritas.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 54a6eb5c

arch/parisc/mm/init.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -608,7 +608,7 @@ void show_mem(void) 
		for (i = 0; i < npmem_ranges; i++) {

		for (i = 0; i < npmem_ranges; i++) {

			zl = node_zonelist(i);

			zl = node_zonelist(i);

			for (j = 0; j < MAX_NR_ZONES; j++) {

			for (j = 0; j < MAX_NR_ZONES; j++) {

				struct zone **z;

				struct zoneref *z;

				struct zone *zone;

				struct zone *zone;





				printk("Zone list for zone %d on node %d: ", j, i);

				printk("Zone list for zone %d on node %d: ", j, i);

fs/buffer.c

+3 −3
Original line number Original line Diff line number Diff line
@@ -360,16 +360,16 @@ void invalidate_bdev(struct block_device *bdev) 
 */

 */

static void free_more_memory(void)

static void free_more_memory(void)

{

{

	struct zone **zones;

	struct zoneref *zrefs;

	int nid;

	int nid;





	wakeup_pdflush(1024);

	wakeup_pdflush(1024);

	yield();

	yield();





	for_each_online_node(nid) {

	for_each_online_node(nid) {

		zones = first_zones_zonelist(node_zonelist(nid, GFP_NOFS),

		zrefs = first_zones_zonelist(node_zonelist(nid, GFP_NOFS),

						gfp_zone(GFP_NOFS));

						gfp_zone(GFP_NOFS));

		if (*zones)

		if (zrefs->zone)

			try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,

			try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,

						GFP_NOFS);

						GFP_NOFS);

	}

	}

include/linux/mmzone.h

+54 −10
Original line number Original line Diff line number Diff line
@@ -468,6 +468,15 @@ struct zonelist_cache { 
struct zonelist_cache;

struct zonelist_cache;

#endif

#endif





/*

 * This struct contains information about a zone in a zonelist. It is stored

 * here to avoid dereferences into large structures and lookups of tables

 */

struct zoneref {

	struct zone *zone;	/* Pointer to actual zone */

	int zone_idx;		/* zone_idx(zoneref->zone) */

};



/*

/*

 * One allocation request operates on a zonelist. A zonelist

 * One allocation request operates on a zonelist. A zonelist

 * is a list of zones, the first one is the 'goal' of the

 * is a list of zones, the first one is the 'goal' of the
@@ -476,11 +485,18 @@ struct zonelist_cache; 
 *

 *

 * If zlcache_ptr is not NULL, then it is just the address of zlcache,

 * If zlcache_ptr is not NULL, then it is just the address of zlcache,

 * as explained above.  If zlcache_ptr is NULL, there is no zlcache.

 * as explained above.  If zlcache_ptr is NULL, there is no zlcache.

 * *

 * To speed the reading of the zonelist, the zonerefs contain the zone index

 * of the entry being read. Helper functions to access information given

 * a struct zoneref are

 *

 * zonelist_zone()	- Return the struct zone * for an entry in _zonerefs

 * zonelist_zone_idx()	- Return the index of the zone for an entry

 * zonelist_node_idx()	- Return the index of the node for an entry

 */

 */



struct zonelist {

struct zonelist {

	struct zonelist_cache *zlcache_ptr;		     // NULL or &zlcache

	struct zonelist_cache *zlcache_ptr;		     // NULL or &zlcache

	struct zone *zones[MAX_ZONES_PER_ZONELIST + 1];      // NULL delimited

	struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];

#ifdef CONFIG_NUMA

#ifdef CONFIG_NUMA

	struct zonelist_cache zlcache;			     // optional ...

	struct zonelist_cache zlcache;			     // optional ...

#endif

#endif
@@ -713,26 +729,52 @@ extern struct zone *next_zone(struct zone *zone); 
	     zone;					\

	     zone;					\

	     zone = next_zone(zone))

	     zone = next_zone(zone))





static inline struct zone *zonelist_zone(struct zoneref *zoneref)

{

	return zoneref->zone;

}



static inline int zonelist_zone_idx(struct zoneref *zoneref)

{

	return zoneref->zone_idx;

}



static inline int zonelist_node_idx(struct zoneref *zoneref)

{

#ifdef CONFIG_NUMA

	/* zone_to_nid not available in this context */

	return zoneref->zone->node;

#else

	return 0;

#endif /* CONFIG_NUMA */

}



static inline void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)

{

	zoneref->zone = zone;

	zoneref->zone_idx = zone_idx(zone);

}



/* Returns the first zone at or below highest_zoneidx in a zonelist */

/* Returns the first zone at or below highest_zoneidx in a zonelist */

static inline struct zone **first_zones_zonelist(struct zonelist *zonelist,

static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,

					enum zone_type highest_zoneidx)

					enum zone_type highest_zoneidx)

{

{

	struct zone **z;

	struct zoneref *z;





	/* Find the first suitable zone to use for the allocation */

	/* Find the first suitable zone to use for the allocation */

	z = zonelist->zones;

	z = zonelist->_zonerefs;

	while (*z && zone_idx(*z) > highest_zoneidx)

	while (zonelist_zone_idx(z) > highest_zoneidx)

		z++;

		z++;





	return z;

	return z;

}

}





/* Returns the next zone at or below highest_zoneidx in a zonelist */

/* Returns the next zone at or below highest_zoneidx in a zonelist */

static inline struct zone **next_zones_zonelist(struct zone **z,

static inline struct zoneref *next_zones_zonelist(struct zoneref *z,

					enum zone_type highest_zoneidx)

					enum zone_type highest_zoneidx)

{

{

	/* Find the next suitable zone to use for the allocation */

	/* Find the next suitable zone to use for the allocation */

	while (*z && zone_idx(*z) > highest_zoneidx)

	while (zonelist_zone_idx(z) > highest_zoneidx)

		z++;

		z++;





	return z;

	return z;
@@ -748,9 +790,11 @@ static inline struct zone **next_zones_zonelist(struct zone **z, 
 * This iterator iterates though all zones at or below a given zone index.

 * This iterator iterates though all zones at or below a given zone index.

 */

 */

#define for_each_zone_zonelist(zone, z, zlist, highidx) \

#define for_each_zone_zonelist(zone, z, zlist, highidx) \

	for (z = first_zones_zonelist(zlist, highidx), zone = *z++;	\

	for (z = first_zones_zonelist(zlist, highidx),			\

					zone = zonelist_zone(z++);	\

		zone;							\

		zone;							\

		z = next_zones_zonelist(z, highidx), zone = *z++)

		z = next_zones_zonelist(z, highidx),			\

					zone = zonelist_zone(z++))





#ifdef CONFIG_SPARSEMEM

#ifdef CONFIG_SPARSEMEM

#include <asm/sparsemem.h>

#include <asm/sparsemem.h>

include/linux/oom.h

+2 −2
Original line number Original line Diff line number Diff line
@@ -23,8 +23,8 @@ enum oom_constraint { 
	CONSTRAINT_MEMORY_POLICY,

	CONSTRAINT_MEMORY_POLICY,

};

};





extern int try_set_zone_oom(struct zonelist *zonelist);

extern int try_set_zone_oom(struct zonelist *zonelist, gfp_t gfp_flags);

extern void clear_zonelist_oom(struct zonelist *zonelist);

extern void clear_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_flags);





extern void out_of_memory(struct zonelist *zonelist, gfp_t gfp_mask, int order);

extern void out_of_memory(struct zonelist *zonelist, gfp_t gfp_mask, int order);

extern int register_oom_notifier(struct notifier_block *nb);

extern int register_oom_notifier(struct notifier_block *nb);

kernel/cpuset.c

+2 −2
Original line number Original line Diff line number Diff line
@@ -1967,8 +1967,8 @@ int cpuset_zonelist_valid_mems_allowed(struct zonelist *zl) 
{

{

	int i;

	int i;





	for (i = 0; zl->zones[i]; i++) {

	for (i = 0; zl->_zonerefs[i].zone; i++) {

		int nid = zone_to_nid(zl->zones[i]);

		int nid = zonelist_node_idx(&zl->_zonerefs[i]);





		if (node_isset(nid, current->mems_allowed))

		if (node_isset(nid, current->mems_allowed))

			return 1;

			return 1;

CRA Git | Maintained and supported by SUSTech CRA and CCSE