mm, compaction: finish pageblock scanning on contention (cb2dcaf0) · Commits · 戴 / test

mm/compaction.c

+34 −56

Original line number	Diff line number	Diff line
		@@ -382,24 +382,25 @@ static bool test_and_set_skip(struct compact_control cc, struct page page,

		/*
		* Compaction requires the taking of some coarse locks that are potentially
		* very heavily contended. For async compaction, back out if the lock cannot
		* be taken immediately. For sync compaction, spin on the lock if needed.
		* very heavily contended. For async compaction, trylock and record if the
		* lock is contended. The lock will still be acquired but compaction will
		* abort when the current block is finished regardless of success rate.
		* Sync compaction acquires the lock.
		*
		* Returns true if the lock is held
		* Returns false if the lock is not held and compaction should abort
		* Always returns true which makes it easier to track lock state in callers.
		*/
		static bool compact_trylock_irqsave(spinlock_t lock, unsigned long flags,
		static bool compact_lock_irqsave(spinlock_t lock, unsigned long flags,
		struct compact_control *cc)
		{
		if (cc->mode == MIGRATE_ASYNC) {
		if (!spin_trylock_irqsave(lock, *flags)) {
		/* Track if the lock is contended in async mode */
		if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
		if (spin_trylock_irqsave(lock, *flags))
		return true;

		cc->contended = true;
		return false;
		}
		} else {
		spin_lock_irqsave(lock, *flags);
		}

		spin_lock_irqsave(lock, *flags);
		return true;
		}

		@@ -432,10 +433,8 @@ static bool compact_unlock_should_abort(spinlock_t *lock,
		}

		if (need_resched()) {
		if (cc->mode == MIGRATE_ASYNC) {
		if (cc->mode == MIGRATE_ASYNC)
		cc->contended = true;
		return true;
		}
		cond_resched();
		}

		@@ -455,10 +454,8 @@ static inline bool compact_should_abort(struct compact_control *cc)
		{
		/* async compaction aborts if contended */
		if (need_resched()) {
		if (cc->mode == MIGRATE_ASYNC) {
		if (cc->mode == MIGRATE_ASYNC)
		cc->contended = true;
		return true;
		}

		cond_resched();
		}
		@@ -535,18 +532,8 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
		* recheck as well.
		*/
		if (!locked) {
		/*
		* The zone lock must be held to isolate freepages.
		* Unfortunately this is a very coarse lock and can be
		* heavily contended if there are parallel allocations
		* or parallel compactions. For async compaction do not
		* spin on the lock and we acquire the lock as late as
		* possible.
		*/
		locked = compact_trylock_irqsave(&cc->zone->lock,
		locked = compact_lock_irqsave(&cc->zone->lock,
		&flags, cc);
		if (!locked)
		break;

		/* Recheck this is a buddy page under lock */
		if (!PageBuddy(page))
		@@ -900,15 +887,9 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,

		/* If we already hold the lock, we can skip some rechecking */
		if (!locked) {
		locked = compact_trylock_irqsave(zone_lru_lock(zone),
		locked = compact_lock_irqsave(zone_lru_lock(zone),
		&flags, cc);

		/* Allow future scanning if the lock is contended */
		if (!locked) {
		clear_pageblock_skip(page);
		break;
		}

		/* Try get exclusive access under lock */
		if (!skip_updated) {
		skip_updated = true;
		@@ -951,9 +932,12 @@ isolate_success:

		/*
		* Avoid isolating too much unless this block is being
		* rescanned (e.g. dirty/writeback pages, parallel allocation).
		* rescanned (e.g. dirty/writeback pages, parallel allocation)
		* or a lock is contended. For contention, isolate quickly to
		* potentially remove one source of contention.
		*/
		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX && !cc->rescan) {
		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX &&
		!cc->rescan && !cc->contended) {
		++low_pfn;
		break;
		}
		@@ -1416,12 +1400,8 @@ static void isolate_freepages(struct compact_control *cc)
		isolate_freepages_block(cc, &isolate_start_pfn, block_end_pfn,
		freelist, false);

		/*
		* If we isolated enough freepages, or aborted due to lock
		* contention, terminate.
		*/
		if ((cc->nr_freepages >= cc->nr_migratepages)
		\|\| cc->contended) {
		/* Are enough freepages isolated? */
		if (cc->nr_freepages >= cc->nr_migratepages) {
		if (isolate_start_pfn >= block_end_pfn) {
		/*
		* Restart at previous pageblock if more
		@@ -1463,12 +1443,7 @@ static struct page compaction_alloc(struct page migratepage,
		struct compact_control cc = (struct compact_control )data;
		struct page *freepage;

		/*
		* Isolate free pages if necessary, and if we are not aborting due to
		* contention.
		*/
		if (list_empty(&cc->freepages)) {
		if (!cc->contended)
		isolate_freepages(cc);

		if (list_empty(&cc->freepages))
		@@ -1733,7 +1708,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
		low_pfn = isolate_migratepages_block(cc, low_pfn,
		block_end_pfn, isolate_mode);

		if (!low_pfn \|\| cc->contended)
		if (!low_pfn)
		return ISOLATE_ABORT;

		/*
		@@ -1763,9 +1738,7 @@ static enum compact_result __compact_finished(struct compact_control *cc)
		{
		unsigned int order;
		const int migratetype = cc->migratetype;

		if (cc->contended \|\| fatal_signal_pending(current))
		return COMPACT_CONTENDED;
		int ret;

		/* Compaction run completes if the migrate and free scanner meet */
		if (compact_scanners_met(cc)) {
		@@ -1800,6 +1773,7 @@ static enum compact_result __compact_finished(struct compact_control *cc)
		return COMPACT_CONTINUE;

		/* Direct compactor: Is a suitable page free? */
		ret = COMPACT_NO_SUITABLE_PAGE;
		for (order = cc->order; order < MAX_ORDER; order++) {
		struct free_area *area = &cc->zone->free_area[order];
		bool can_steal;
		@@ -1839,11 +1813,15 @@ static enum compact_result __compact_finished(struct compact_control *cc)
		return COMPACT_SUCCESS;
		}

		return COMPACT_CONTINUE;
		ret = COMPACT_CONTINUE;
		break;
		}
		}

		return COMPACT_NO_SUITABLE_PAGE;
		if (cc->contended \|\| fatal_signal_pending(current))
		ret = COMPACT_CONTENDED;

		return ret;
		}

		static enum compact_result compact_finished(struct compact_control *cc)

Admin message