mm, page_alloc: inline pageblock lookup in page free fast paths (0b423ca2) · Commits · 戴 / test

include/linux/mmzone.h

+0 −7

Original line number	Original line	Diff line number	Diff line
	@@ -85,13 +85,6 @@ extern int page_group_by_mobility_disabled;
	get_pfnblock_flags_mask(page, page_to_pfn(page), \		get_pfnblock_flags_mask(page, page_to_pfn(page), \
	PB_migrate_end, MIGRATETYPE_MASK)		PB_migrate_end, MIGRATETYPE_MASK)

	static inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
	{
	BUILD_BUG_ON(PB_migrate_end - PB_migrate != 2);
	return get_pfnblock_flags_mask(page, pfn, PB_migrate_end,
	MIGRATETYPE_MASK);
	}

	struct free_area {		struct free_area {
	struct list_head free_list[MIGRATE_TYPES];		struct list_head free_list[MIGRATE_TYPES];
	unsigned long nr_free;		unsigned long nr_free;

mm/page_alloc.c

+100 −88

Original line number	Original line	Diff line number	Diff line
	@@ -352,6 +352,106 @@ static inline bool update_defer_init(pg_data_t *pgdat,
	}		}
	#endif		#endif

			/* Return a pointer to the bitmap storing bits affecting a block of pages */
			static inline unsigned long get_pageblock_bitmap(struct page page,
			unsigned long pfn)
			{
			#ifdef CONFIG_SPARSEMEM
			return __pfn_to_section(pfn)->pageblock_flags;
			#else
			return page_zone(page)->pageblock_flags;
			#endif /* CONFIG_SPARSEMEM */
			}

			static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
			{
			#ifdef CONFIG_SPARSEMEM
			pfn &= (PAGES_PER_SECTION-1);
			return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
			#else
			pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
			return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
			#endif /* CONFIG_SPARSEMEM */
			}

			/**
			* get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
			* @page: The page within the block of interest
			* @pfn: The target page frame number
			* @end_bitidx: The last bit of interest to retrieve
			* @mask: mask of bits that the caller is interested in
			*
			* Return: pageblock_bits flags
			*/
			static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
			unsigned long pfn,
			unsigned long end_bitidx,
			unsigned long mask)
			{
			unsigned long *bitmap;
			unsigned long bitidx, word_bitidx;
			unsigned long word;

			bitmap = get_pageblock_bitmap(page, pfn);
			bitidx = pfn_to_bitidx(page, pfn);
			word_bitidx = bitidx / BITS_PER_LONG;
			bitidx &= (BITS_PER_LONG-1);

			word = bitmap[word_bitidx];
			bitidx += end_bitidx;
			return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
			}

			unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
			unsigned long end_bitidx,
			unsigned long mask)
			{
			return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
			}

			static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
			{
			return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
			}

			/**
			* set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
			* @page: The page within the block of interest
			* @flags: The flags to set
			* @pfn: The target page frame number
			* @end_bitidx: The last bit of interest
			* @mask: mask of bits that the caller is interested in
			*/
			void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
			unsigned long pfn,
			unsigned long end_bitidx,
			unsigned long mask)
			{
			unsigned long *bitmap;
			unsigned long bitidx, word_bitidx;
			unsigned long old_word, word;

			BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);

			bitmap = get_pageblock_bitmap(page, pfn);
			bitidx = pfn_to_bitidx(page, pfn);
			word_bitidx = bitidx / BITS_PER_LONG;
			bitidx &= (BITS_PER_LONG-1);

			VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);

			bitidx += end_bitidx;
			mask <<= (BITS_PER_LONG - bitidx - 1);
			flags <<= (BITS_PER_LONG - bitidx - 1);

			word = READ_ONCE(bitmap[word_bitidx]);
			for (;;) {
			old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) \| flags);
			if (word == old_word)
			break;
			word = old_word;
			}
			}

	void set_pageblock_migratetype(struct page *page, int migratetype)		void set_pageblock_migratetype(struct page *page, int migratetype)
	{		{
	@@ -6831,94 +6931,6 @@ void __init alloc_large_system_hash(const char tablename,
	return table;		return table;
	}		}

	/* Return a pointer to the bitmap storing bits affecting a block of pages */
	static inline unsigned long get_pageblock_bitmap(struct page page,
	unsigned long pfn)
	{
	#ifdef CONFIG_SPARSEMEM
	return __pfn_to_section(pfn)->pageblock_flags;
	#else
	return page_zone(page)->pageblock_flags;
	#endif /* CONFIG_SPARSEMEM */
	}

	static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
	{
	#ifdef CONFIG_SPARSEMEM
	pfn &= (PAGES_PER_SECTION-1);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
	#else
	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
	#endif /* CONFIG_SPARSEMEM */
	}

	/**
	* get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
	* @page: The page within the block of interest
	* @pfn: The target page frame number
	* @end_bitidx: The last bit of interest to retrieve
	* @mask: mask of bits that the caller is interested in
	*
	* Return: pageblock_bits flags
	*/
	unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
	unsigned long end_bitidx,
	unsigned long mask)
	{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long word;

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	word = bitmap[word_bitidx];
	bitidx += end_bitidx;
	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
	}

	/**
	* set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
	* @page: The page within the block of interest
	* @flags: The flags to set
	* @pfn: The target page frame number
	* @end_bitidx: The last bit of interest
	* @mask: mask of bits that the caller is interested in
	*/
	void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
	unsigned long pfn,
	unsigned long end_bitidx,
	unsigned long mask)
	{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long old_word, word;

	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);

	bitidx += end_bitidx;
	mask <<= (BITS_PER_LONG - bitidx - 1);
	flags <<= (BITS_PER_LONG - bitidx - 1);

	word = READ_ONCE(bitmap[word_bitidx]);
	for (;;) {
	old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) \| flags);
	if (word == old_word)
	break;
	word = old_word;
	}
	}

	/*		/*
	* This function checks whether pageblock includes unmovable pages or not.		* This function checks whether pageblock includes unmovable pages or not.
	* If @count is not zero, it is okay to include less @count unmovable pages		* If @count is not zero, it is okay to include less @count unmovable pages

mm/page_owner.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -143,7 +143,7 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn,
	goto err;		goto err;

	/* Print information relevant to grouping pages by mobility */		/* Print information relevant to grouping pages by mobility */
	pageblock_mt = get_pfnblock_migratetype(page, pfn);		pageblock_mt = get_pageblock_migratetype(page);
	page_mt = gfpflags_to_migratetype(page_ext->gfp_mask);		page_mt = gfpflags_to_migratetype(page_ext->gfp_mask);
	ret += snprintf(kbuf + ret, count - ret,		ret += snprintf(kbuf + ret, count - ret,
	"PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",		"PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",

mm/vmstat.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -1041,7 +1041,7 @@ static void pagetypeinfo_showmixedcount_print(struct seq_file *m,
	block_end_pfn = min(block_end_pfn, end_pfn);		block_end_pfn = min(block_end_pfn, end_pfn);

	page = pfn_to_page(pfn);		page = pfn_to_page(pfn);
	pageblock_mt = get_pfnblock_migratetype(page, pfn);		pageblock_mt = get_pageblock_migratetype(page);

	for (; pfn < block_end_pfn; pfn++) {		for (; pfn < block_end_pfn; pfn++) {
	if (!pfn_valid_within(pfn))		if (!pfn_valid_within(pfn))

Admin message