Commit 90572890 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

mm: numa: Change page last {nid,pid} into {cpu,pid} 



Change the per page last fault tracking to use cpu,pid instead of
nid,pid. This will allow us to try and lookup the alternate task more
easily. Note that even though it is the cpu that is store in the page
flags that the mpol_misplaced decision is still based on the node.

Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarMel Gorman <mgorman@suse.de>
Reviewed-by: default avatarRik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Link: http://lkml.kernel.org/r/1381141781-10992-43-git-send-email-mgorman@suse.de


[ Fixed build failure on 32-bit systems. ]
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent e1dda8a7

include/linux/mm.h

+50 −40
Original line number Diff line number Diff line
@@ -581,11 +581,11 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 
 * sets it, so none of the operations on it need to be atomic.

 */



/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NIDPID] | ... | FLAGS | */

/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */

#define SECTIONS_PGOFF		((sizeof(unsigned long)*8) - SECTIONS_WIDTH)

#define NODES_PGOFF		(SECTIONS_PGOFF - NODES_WIDTH)

#define ZONES_PGOFF		(NODES_PGOFF - ZONES_WIDTH)

#define LAST_NIDPID_PGOFF	(ZONES_PGOFF - LAST_NIDPID_WIDTH)

#define LAST_CPUPID_PGOFF	(ZONES_PGOFF - LAST_CPUPID_WIDTH)



/*

 * Define the bit shifts to access each section.  For non-existent
@@ -595,7 +595,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 
#define SECTIONS_PGSHIFT	(SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))

#define NODES_PGSHIFT		(NODES_PGOFF * (NODES_WIDTH != 0))

#define ZONES_PGSHIFT		(ZONES_PGOFF * (ZONES_WIDTH != 0))

#define LAST_NIDPID_PGSHIFT	(LAST_NIDPID_PGOFF * (LAST_NIDPID_WIDTH != 0))

#define LAST_CPUPID_PGSHIFT	(LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))



/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */

#ifdef NODE_NOT_IN_PAGE_FLAGS
@@ -617,7 +617,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 
#define ZONES_MASK		((1UL << ZONES_WIDTH) - 1)

#define NODES_MASK		((1UL << NODES_WIDTH) - 1)

#define SECTIONS_MASK		((1UL << SECTIONS_WIDTH) - 1)

#define LAST_NIDPID_MASK	((1UL << LAST_NIDPID_WIDTH) - 1)

#define LAST_CPUPID_MASK	((1UL << LAST_CPUPID_WIDTH) - 1)

#define ZONEID_MASK		((1UL << ZONEID_SHIFT) - 1)



static inline enum zone_type page_zonenum(const struct page *page)
@@ -661,96 +661,106 @@ static inline int page_to_nid(const struct page *page) 
#endif



#ifdef CONFIG_NUMA_BALANCING

static inline int nid_pid_to_nidpid(int nid, int pid)

static inline int cpu_pid_to_cpupid(int cpu, int pid)

{

	return ((nid & LAST__NID_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);

	return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);

}



static inline int nidpid_to_pid(int nidpid)

static inline int cpupid_to_pid(int cpupid)

{

	return nidpid & LAST__PID_MASK;

	return cpupid & LAST__PID_MASK;

}



static inline int nidpid_to_nid(int nidpid)

static inline int cpupid_to_cpu(int cpupid)

{

	return (nidpid >> LAST__PID_SHIFT) & LAST__NID_MASK;

	return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;

}



static inline bool nidpid_pid_unset(int nidpid)

static inline int cpupid_to_nid(int cpupid)

{

	return nidpid_to_pid(nidpid) == (-1 & LAST__PID_MASK);

	return cpu_to_node(cpupid_to_cpu(cpupid));

}



static inline bool nidpid_nid_unset(int nidpid)

static inline bool cpupid_pid_unset(int cpupid)

{

	return nidpid_to_nid(nidpid) == (-1 & LAST__NID_MASK);

	return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);

}



#ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS

static inline int page_nidpid_xchg_last(struct page *page, int nid)

static inline bool cpupid_cpu_unset(int cpupid)

{

	return xchg(&page->_last_nidpid, nid);

	return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);

}



static inline int page_nidpid_last(struct page *page)

#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)

{

	return page->_last_nidpid;

	return xchg(&page->_last_cpupid, cpupid);

}

static inline void page_nidpid_reset_last(struct page *page)



static inline int page_cpupid_last(struct page *page)

{

	return page->_last_cpupid;

}

static inline void page_cpupid_reset_last(struct page *page)

{

	page->_last_nidpid = -1;

	page->_last_cpupid = -1;

}

#else

static inline int page_nidpid_last(struct page *page)

static inline int page_cpupid_last(struct page *page)

{

	return (page->flags >> LAST_NIDPID_PGSHIFT) & LAST_NIDPID_MASK;

	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;

}



extern int page_nidpid_xchg_last(struct page *page, int nidpid);

extern int page_cpupid_xchg_last(struct page *page, int cpupid);



static inline void page_nidpid_reset_last(struct page *page)

static inline void page_cpupid_reset_last(struct page *page)

{

	int nidpid = (1 << LAST_NIDPID_SHIFT) - 1;

	int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;



	page->flags &= ~(LAST_NIDPID_MASK << LAST_NIDPID_PGSHIFT);

	page->flags |= (nidpid & LAST_NIDPID_MASK) << LAST_NIDPID_PGSHIFT;

	page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);

	page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;

}

#endif /* LAST_NIDPID_NOT_IN_PAGE_FLAGS */

#else

static inline int page_nidpid_xchg_last(struct page *page, int nidpid)

#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */

#else /* !CONFIG_NUMA_BALANCING */

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)

{

	return page_to_nid(page);

	return page_to_nid(page); /* XXX */

}



static inline int page_nidpid_last(struct page *page)

static inline int page_cpupid_last(struct page *page)

{

	return page_to_nid(page);

	return page_to_nid(page); /* XXX */

}



static inline int nidpid_to_nid(int nidpid)

static inline int cpupid_to_nid(int cpupid)

{

	return -1;

}



static inline int nidpid_to_pid(int nidpid)

static inline int cpupid_to_pid(int cpupid)

{

	return -1;

}



static inline int nid_pid_to_nidpid(int nid, int pid)

static inline int cpupid_to_cpu(int cpupid)

{

	return -1;

}



static inline bool nidpid_pid_unset(int nidpid)

static inline int cpu_pid_to_cpupid(int nid, int pid)

{

	return -1;

}



static inline bool cpupid_pid_unset(int cpupid)

{

	return 1;

}



static inline void page_nidpid_reset_last(struct page *page)

static inline void page_cpupid_reset_last(struct page *page)

{

}

#endif

#endif /* CONFIG_NUMA_BALANCING */



static inline struct zone *page_zone(const struct page *page)

{

include/linux/mm_types.h

+2 −2
Original line number Diff line number Diff line
@@ -174,8 +174,8 @@ struct page { 
	void *shadow;

#endif



#ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS

	int _last_nidpid;

#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS

	int _last_cpupid;

#endif

}

/*

include/linux/page-flags-layout.h

+11 −11
Original line number Diff line number Diff line
@@ -39,9 +39,9 @@ 
 * lookup is necessary.

 *

 * No sparsemem or sparsemem vmemmap: |       NODE     | ZONE |             ... | FLAGS |

 *      " plus space for last_nidpid: |       NODE     | ZONE | LAST_NIDPID ... | FLAGS |

 *      " plus space for last_cpupid: |       NODE     | ZONE | LAST_CPUPID ... | FLAGS |

 * classic sparse with space for node:| SECTION | NODE | ZONE |             ... | FLAGS |

 *      " plus space for last_nidpid: | SECTION | NODE | ZONE | LAST_NIDPID ... | FLAGS |

 *      " plus space for last_cpupid: | SECTION | NODE | ZONE | LAST_CPUPID ... | FLAGS |

 * classic sparse no space for node:  | SECTION |     ZONE    | ... | FLAGS |

 */

#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
@@ -65,18 +65,18 @@ 
#define LAST__PID_SHIFT 8

#define LAST__PID_MASK  ((1 << LAST__PID_SHIFT)-1)



#define LAST__NID_SHIFT NODES_SHIFT

#define LAST__NID_MASK  ((1 << LAST__NID_SHIFT)-1)

#define LAST__CPU_SHIFT NR_CPUS_BITS

#define LAST__CPU_MASK  ((1 << LAST__CPU_SHIFT)-1)



#define LAST_NIDPID_SHIFT (LAST__PID_SHIFT+LAST__NID_SHIFT)

#define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT)

#else

#define LAST_NIDPID_SHIFT 0

#define LAST_CPUPID_SHIFT 0

#endif



#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_NIDPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS

#define LAST_NIDPID_WIDTH LAST_NIDPID_SHIFT

#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS

#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT

#else

#define LAST_NIDPID_WIDTH 0

#define LAST_CPUPID_WIDTH 0

#endif



/*
@@ -87,8 +87,8 @@ 
#define NODE_NOT_IN_PAGE_FLAGS

#endif



#if defined(CONFIG_NUMA_BALANCING) && LAST_NIDPID_WIDTH == 0

#define LAST_NIDPID_NOT_IN_PAGE_FLAGS

#if defined(CONFIG_NUMA_BALANCING) && LAST_CPUPID_WIDTH == 0

#define LAST_CPUPID_NOT_IN_PAGE_FLAGS

#endif



#endif /* _LINUX_PAGE_FLAGS_LAYOUT */

kernel/bounds.c

+4 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
#include <linux/mmzone.h>

#include <linux/kbuild.h>

#include <linux/page_cgroup.h>

#include <linux/log2.h>



void foo(void)

{
@@ -17,5 +18,8 @@ void foo(void) 
	DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS);

	DEFINE(MAX_NR_ZONES, __MAX_NR_ZONES);

	DEFINE(NR_PCG_FLAGS, __NR_PCG_FLAGS);

#ifdef CONFIG_SMP

	DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));

#endif

	/* End of constants */

}

kernel/sched/fair.c

+3 −3
Original line number Diff line number Diff line
@@ -1210,7 +1210,7 @@ static void task_numa_placement(struct task_struct *p) 
/*

 * Got a PROT_NONE fault for a page on @node.

 */

void task_numa_fault(int last_nidpid, int node, int pages, bool migrated)

void task_numa_fault(int last_cpupid, int node, int pages, bool migrated)

{

	struct task_struct *p = current;

	int priv;
@@ -1226,8 +1226,8 @@ void task_numa_fault(int last_nidpid, int node, int pages, bool migrated) 
	 * First accesses are treated as private, otherwise consider accesses

	 * to be private if the accessing pid has not changed

	 */

	if (!nidpid_pid_unset(last_nidpid))

		priv = ((p->pid & LAST__PID_MASK) == nidpid_to_pid(last_nidpid));

	if (!cpupid_pid_unset(last_cpupid))

		priv = ((p->pid & LAST__PID_MASK) == cpupid_to_pid(last_cpupid));

	else

		priv = 1;

CRA Git | Maintained and supported by SUSTech CRA and CCSE