Commit 62906027 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Linus Torvalds
Browse files

mm: add PageWaiters indicating tasks are waiting for a page bit 



Add a new page flag, PageWaiters, to indicate the page waitqueue has
tasks waiting. This can be tested rather than testing waitqueue_active
which requires another cacheline load.

This bit is always set when the page has tasks on page_waitqueue(page),
and is set and cleared under the waitqueue lock. It may be set when
there are no tasks on the waitqueue, which will cause a harmless extra
wakeup check that will clears the bit.

The generic bit-waitqueue infrastructure is no longer used for pages.
Instead, waitqueues are used directly with a custom key type. The
generic code was not flexible enough to have PageWaiters manipulation
under the waitqueue lock (which simplifies concurrency).

This improves the performance of page lock intensive microbenchmarks by
2-3%.

Putting two bits in the same word opens the opportunity to remove the
memory barrier between clearing the lock bit and testing the waiters
bit, after some work on the arch primitives (e.g., ensuring memory
operand widths match and cover both bits).

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Bob Peterson <rpeterso@redhat.com>
Cc: Steven Whitehouse <swhiteho@redhat.com>
Cc: Andrew Lutomirski <luto@kernel.org>
Cc: Andreas Gruenbacher <agruenba@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 6326fec1

include/linux/mm.h

+2 −0
Original line number Diff line number Diff line
@@ -1758,6 +1758,8 @@ static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) 
	return ptl;

}



extern void __init pagecache_init(void);



extern void free_area_init(unsigned long * zones_size);

extern void free_area_init_node(int nid, unsigned long * zones_size,

		unsigned long zone_start_pfn, unsigned long *zholes_size);

include/linux/page-flags.h

+9 −0
Original line number Diff line number Diff line
@@ -73,6 +73,7 @@ 
 */

enum pageflags {

	PG_locked,		/* Page is locked. Don't touch. */

	PG_waiters,		/* Page has waiters, check its waitqueue */

	PG_error,

	PG_referenced,

	PG_uptodate,
@@ -169,6 +170,9 @@ static __always_inline int PageCompound(struct page *page) 
 *     for compound page all operations related to the page flag applied to

 *     head page.

 *

 * PF_ONLY_HEAD:

 *     for compound page, callers only ever operate on the head page.

 *

 * PF_NO_TAIL:

 *     modifications of the page flag must be done on small or head pages,

 *     checks can be done on tail pages too.
@@ -178,6 +182,9 @@ static __always_inline int PageCompound(struct page *page) 
 */

#define PF_ANY(page, enforce)	page

#define PF_HEAD(page, enforce)	compound_head(page)

#define PF_ONLY_HEAD(page, enforce) ({					\

		VM_BUG_ON_PGFLAGS(PageTail(page), page);		\

		page;})

#define PF_NO_TAIL(page, enforce) ({					\

		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\

		compound_head(page);})
@@ -255,6 +262,7 @@ static inline int TestClearPage##uname(struct page *page) { return 0; } 
	TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)



__PAGEFLAG(Locked, locked, PF_NO_TAIL)

PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)

PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)

PAGEFLAG(Referenced, referenced, PF_HEAD)

	TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
@@ -743,6 +751,7 @@ static inline int page_has_private(struct page *page) 


#undef PF_ANY

#undef PF_HEAD

#undef PF_ONLY_HEAD

#undef PF_NO_TAIL

#undef PF_NO_COMPOUND

#endif /* !__GENERATING_BOUNDS_H */

include/linux/pagemap.h

+11 −12
Original line number Diff line number Diff line
@@ -486,22 +486,14 @@ static inline int lock_page_or_retry(struct page *page, struct mm_struct *mm, 
 * and for filesystems which need to wait on PG_private.

 */

extern void wait_on_page_bit(struct page *page, int bit_nr);



extern int wait_on_page_bit_killable(struct page *page, int bit_nr);

extern int wait_on_page_bit_killable_timeout(struct page *page,

					     int bit_nr, unsigned long timeout);



static inline int wait_on_page_locked_killable(struct page *page)

{

	if (!PageLocked(page))

		return 0;

	return wait_on_page_bit_killable(compound_head(page), PG_locked);

}

extern void wake_up_page_bit(struct page *page, int bit_nr);



extern wait_queue_head_t *page_waitqueue(struct page *page);

static inline void wake_up_page(struct page *page, int bit)

{

	__wake_up_bit(page_waitqueue(page), &page->flags, bit);

	if (!PageWaiters(page))

		return;

	wake_up_page_bit(page, bit);

}



/*
@@ -517,6 +509,13 @@ static inline void wait_on_page_locked(struct page *page) 
		wait_on_page_bit(compound_head(page), PG_locked);

}



static inline int wait_on_page_locked_killable(struct page *page)

{

	if (!PageLocked(page))

		return 0;

	return wait_on_page_bit_killable(compound_head(page), PG_locked);

}



/* 

 * Wait for a page to complete writeback

 */

include/linux/writeback.h

+0 −1
Original line number Diff line number Diff line
@@ -375,7 +375,6 @@ void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty); 
unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh);



void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time);

void page_writeback_init(void);

void balance_dirty_pages_ratelimited(struct address_space *mapping);

bool wb_over_bg_thresh(struct bdi_writeback *wb);

include/trace/events/mmflags.h

+1 −0
Original line number Diff line number Diff line
@@ -81,6 +81,7 @@ 


#define __def_pageflag_names						\

	{1UL << PG_locked,		"locked"	},		\

	{1UL << PG_waiters,		"waiters"	},		\

	{1UL << PG_error,		"error"		},		\

	{1UL << PG_referenced,		"referenced"	},		\

	{1UL << PG_uptodate,		"uptodate"	},		\

CRA Git | Maintained and supported by SUSTech CRA and CCSE