mm: page cache: store only head pages in i_pages (5fd4ca2d) · Commits · 戴 / test

include/linux/pagemap.h

+13 −0

Original line number	Original line	Diff line number	Diff line
	@@ -333,6 +333,19 @@ static inline struct page grab_cache_page_nowait(struct address_space mapping,
	mapping_gfp_mask(mapping));		mapping_gfp_mask(mapping));
	}		}

			static inline struct page find_subpage(struct page page, pgoff_t offset)
			{
			unsigned long mask;

			if (PageHuge(page))
			return page;

			VM_BUG_ON_PAGE(PageTail(page), page);

			mask = (1UL << compound_order(page)) - 1;
			return page + (offset & mask);
			}

	struct page find_get_entry(struct address_space mapping, pgoff_t offset);		struct page find_get_entry(struct address_space mapping, pgoff_t offset);
	struct page find_lock_entry(struct address_space mapping, pgoff_t offset);		struct page find_lock_entry(struct address_space mapping, pgoff_t offset);
	unsigned find_get_entries(struct address_space *mapping, pgoff_t start,		unsigned find_get_entries(struct address_space *mapping, pgoff_t start,

mm/filemap.c

+62 −97

Original line number	Original line	Diff line number	Diff line
	@@ -279,11 +279,11 @@ EXPORT_SYMBOL(delete_from_page_cache);
	* @pvec: pagevec with pages to delete		* @pvec: pagevec with pages to delete
	*		*
	* The function walks over mapping->i_pages and removes pages passed in @pvec		* The function walks over mapping->i_pages and removes pages passed in @pvec
	* from the mapping. The function expects @pvec to be sorted by page index.		* from the mapping. The function expects @pvec to be sorted by page index
			* and is optimised for it to be dense.
	* It tolerates holes in @pvec (mapping entries at those indices are not		* It tolerates holes in @pvec (mapping entries at those indices are not
	* modified). The function expects only THP head pages to be present in the		* modified). The function expects only THP head pages to be present in the
	* @pvec and takes care to delete all corresponding tail pages from the		* @pvec.
	* mapping as well.
	*		*
	* The function expects the i_pages lock to be held.		* The function expects the i_pages lock to be held.
	*/		*/
	@@ -292,40 +292,44 @@ static void page_cache_delete_batch(struct address_space *mapping,
	{		{
	XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);		XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);
	int total_pages = 0;		int total_pages = 0;
	int i = 0, tail_pages = 0;		int i = 0;
	struct page *page;		struct page *page;

	mapping_set_update(&xas, mapping);		mapping_set_update(&xas, mapping);
	xas_for_each(&xas, page, ULONG_MAX) {		xas_for_each(&xas, page, ULONG_MAX) {
	if (i >= pagevec_count(pvec) && !tail_pages)		if (i >= pagevec_count(pvec))
	break;		break;

			/* A swap/dax/shadow entry got inserted? Skip it. */
	if (xa_is_value(page))		if (xa_is_value(page))
	continue;		continue;
	if (!tail_pages) {
	/*		/*
	* Some page got inserted in our range? Skip it. We		* A page got inserted in our range? Skip it. We have our
	* have our pages locked so they are protected from		* pages locked so they are protected from being removed.
	* being removed.		* If we see a page whose index is higher than ours, it
			* means our page has been removed, which shouldn't be
			* possible because we're holding the PageLock.
	*/		*/
	if (page != pvec->pages[i]) {		if (page != pvec->pages[i]) {
	VM_BUG_ON_PAGE(page->index >		VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index,
	pvec->pages[i]->index, page);		page);
	continue;		continue;
	}		}

	WARN_ON_ONCE(!PageLocked(page));		WARN_ON_ONCE(!PageLocked(page));
	if (PageTransHuge(page) && !PageHuge(page))
	tail_pages = HPAGE_PMD_NR - 1;		if (page->index == xas.xa_index)
	page->mapping = NULL;		page->mapping = NULL;
			/* Leave page->index set: truncation lookup relies on it */

	/*		/*
	* Leave page->index set: truncation lookup relies		* Move to the next page in the vector if this is a regular
	* upon it		* page or the index is of the last sub-page of this compound
			* page.
	*/		*/
			if (page->index + (1UL << compound_order(page)) - 1 ==
			xas.xa_index)
	i++;		i++;
	} else {
	VM_BUG_ON_PAGE(page->index + HPAGE_PMD_NR - tail_pages
	!= pvec->pages[i]->index, page);
	tail_pages--;
	}
	xas_store(&xas, NULL);		xas_store(&xas, NULL);
	total_pages++;		total_pages++;
	}		}
	@@ -1491,7 +1495,7 @@ EXPORT_SYMBOL(page_cache_prev_miss);
	struct page find_get_entry(struct address_space mapping, pgoff_t offset)		struct page find_get_entry(struct address_space mapping, pgoff_t offset)
	{		{
	XA_STATE(xas, &mapping->i_pages, offset);		XA_STATE(xas, &mapping->i_pages, offset);
	struct page head, page;		struct page *page;

	rcu_read_lock();		rcu_read_lock();
	repeat:		repeat:
	@@ -1506,25 +1510,19 @@ repeat:
	if (!page \|\| xa_is_value(page))		if (!page \|\| xa_is_value(page))
	goto out;		goto out;

	head = compound_head(page);		if (!page_cache_get_speculative(page))
	if (!page_cache_get_speculative(head))
	goto repeat;		goto repeat;

	/* The page was split under us? */
	if (compound_head(page) != head) {
	put_page(head);
	goto repeat;
	}

	/*		/*
	* Has the page moved?		* Has the page moved or been split?
	* This is part of the lockless pagecache protocol. See		* This is part of the lockless pagecache protocol. See
	* include/linux/pagemap.h for details.		* include/linux/pagemap.h for details.
	*/		*/
	if (unlikely(page != xas_reload(&xas))) {		if (unlikely(page != xas_reload(&xas))) {
	put_page(head);		put_page(page);
	goto repeat;		goto repeat;
	}		}
			page = find_subpage(page, offset);
	out:		out:
	rcu_read_unlock();		rcu_read_unlock();

	@@ -1706,7 +1704,6 @@ unsigned find_get_entries(struct address_space *mapping,

	rcu_read_lock();		rcu_read_lock();
	xas_for_each(&xas, page, ULONG_MAX) {		xas_for_each(&xas, page, ULONG_MAX) {
	struct page *head;
	if (xas_retry(&xas, page))		if (xas_retry(&xas, page))
	continue;		continue;
	/*		/*
	@@ -1717,17 +1714,13 @@ unsigned find_get_entries(struct address_space *mapping,
	if (xa_is_value(page))		if (xa_is_value(page))
	goto export;		goto export;

	head = compound_head(page);		if (!page_cache_get_speculative(page))
	if (!page_cache_get_speculative(head))
	goto retry;		goto retry;

	/* The page was split under us? */		/* Has the page moved or been split? */
	if (compound_head(page) != head)
	goto put_page;

	/* Has the page moved? */
	if (unlikely(page != xas_reload(&xas)))		if (unlikely(page != xas_reload(&xas)))
	goto put_page;		goto put_page;
			page = find_subpage(page, xas.xa_index);

	export:		export:
	indices[ret] = xas.xa_index;		indices[ret] = xas.xa_index;
	@@ -1736,7 +1729,7 @@ export:
	break;		break;
	continue;		continue;
	put_page:		put_page:
	put_page(head);		put_page(page);
	retry:		retry:
	xas_reset(&xas);		xas_reset(&xas);
	}		}
	@@ -1778,33 +1771,27 @@ unsigned find_get_pages_range(struct address_space mapping, pgoff_t start,

	rcu_read_lock();		rcu_read_lock();
	xas_for_each(&xas, page, end) {		xas_for_each(&xas, page, end) {
	struct page *head;
	if (xas_retry(&xas, page))		if (xas_retry(&xas, page))
	continue;		continue;
	/* Skip over shadow, swap and DAX entries */		/* Skip over shadow, swap and DAX entries */
	if (xa_is_value(page))		if (xa_is_value(page))
	continue;		continue;

	head = compound_head(page);		if (!page_cache_get_speculative(page))
	if (!page_cache_get_speculative(head))
	goto retry;		goto retry;

	/* The page was split under us? */		/* Has the page moved or been split? */
	if (compound_head(page) != head)
	goto put_page;

	/* Has the page moved? */
	if (unlikely(page != xas_reload(&xas)))		if (unlikely(page != xas_reload(&xas)))
	goto put_page;		goto put_page;

	pages[ret] = page;		pages[ret] = find_subpage(page, xas.xa_index);
	if (++ret == nr_pages) {		if (++ret == nr_pages) {
	*start = xas.xa_index + 1;		*start = xas.xa_index + 1;
	goto out;		goto out;
	}		}
	continue;		continue;
	put_page:		put_page:
	put_page(head);		put_page(page);
	retry:		retry:
	xas_reset(&xas);		xas_reset(&xas);
	}		}
	@@ -1849,7 +1836,6 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,

	rcu_read_lock();		rcu_read_lock();
	for (page = xas_load(&xas); page; page = xas_next(&xas)) {		for (page = xas_load(&xas); page; page = xas_next(&xas)) {
	struct page *head;
	if (xas_retry(&xas, page))		if (xas_retry(&xas, page))
	continue;		continue;
	/*		/*
	@@ -1859,24 +1845,19 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
	if (xa_is_value(page))		if (xa_is_value(page))
	break;		break;

	head = compound_head(page);		if (!page_cache_get_speculative(page))
	if (!page_cache_get_speculative(head))
	goto retry;		goto retry;

	/* The page was split under us? */		/* Has the page moved or been split? */
	if (compound_head(page) != head)
	goto put_page;

	/* Has the page moved? */
	if (unlikely(page != xas_reload(&xas)))		if (unlikely(page != xas_reload(&xas)))
	goto put_page;		goto put_page;

	pages[ret] = page;		pages[ret] = find_subpage(page, xas.xa_index);
	if (++ret == nr_pages)		if (++ret == nr_pages)
	break;		break;
	continue;		continue;
	put_page:		put_page:
	put_page(head);		put_page(page);
	retry:		retry:
	xas_reset(&xas);		xas_reset(&xas);
	}		}
	@@ -1912,7 +1893,6 @@ unsigned find_get_pages_range_tag(struct address_space mapping, pgoff_t index,

	rcu_read_lock();		rcu_read_lock();
	xas_for_each_marked(&xas, page, end, tag) {		xas_for_each_marked(&xas, page, end, tag) {
	struct page *head;
	if (xas_retry(&xas, page))		if (xas_retry(&xas, page))
	continue;		continue;
	/*		/*
	@@ -1923,26 +1903,21 @@ unsigned find_get_pages_range_tag(struct address_space mapping, pgoff_t index,
	if (xa_is_value(page))		if (xa_is_value(page))
	continue;		continue;

	head = compound_head(page);		if (!page_cache_get_speculative(page))
	if (!page_cache_get_speculative(head))
	goto retry;		goto retry;

	/* The page was split under us? */		/* Has the page moved or been split? */
	if (compound_head(page) != head)
	goto put_page;

	/* Has the page moved? */
	if (unlikely(page != xas_reload(&xas)))		if (unlikely(page != xas_reload(&xas)))
	goto put_page;		goto put_page;

	pages[ret] = page;		pages[ret] = find_subpage(page, xas.xa_index);
	if (++ret == nr_pages) {		if (++ret == nr_pages) {
	*index = xas.xa_index + 1;		*index = xas.xa_index + 1;
	goto out;		goto out;
	}		}
	continue;		continue;
	put_page:		put_page:
	put_page(head);		put_page(page);
	retry:		retry:
	xas_reset(&xas);		xas_reset(&xas);
	}		}
	@@ -1991,7 +1966,6 @@ unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,

	rcu_read_lock();		rcu_read_lock();
	xas_for_each_marked(&xas, page, ULONG_MAX, tag) {		xas_for_each_marked(&xas, page, ULONG_MAX, tag) {
	struct page *head;
	if (xas_retry(&xas, page))		if (xas_retry(&xas, page))
	continue;		continue;
	/*		/*
	@@ -2002,17 +1976,13 @@ unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
	if (xa_is_value(page))		if (xa_is_value(page))
	goto export;		goto export;

	head = compound_head(page);		if (!page_cache_get_speculative(page))
	if (!page_cache_get_speculative(head))
	goto retry;		goto retry;

	/* The page was split under us? */		/* Has the page moved or been split? */
	if (compound_head(page) != head)
	goto put_page;

	/* Has the page moved? */
	if (unlikely(page != xas_reload(&xas)))		if (unlikely(page != xas_reload(&xas)))
	goto put_page;		goto put_page;
			page = find_subpage(page, xas.xa_index);

	export:		export:
	indices[ret] = xas.xa_index;		indices[ret] = xas.xa_index;
	@@ -2021,7 +1991,7 @@ export:
	break;		break;
	continue;		continue;
	put_page:		put_page:
	put_page(head);		put_page(page);
	retry:		retry:
	xas_reset(&xas);		xas_reset(&xas);
	}		}
	@@ -2691,7 +2661,7 @@ void filemap_map_pages(struct vm_fault *vmf,
	pgoff_t last_pgoff = start_pgoff;		pgoff_t last_pgoff = start_pgoff;
	unsigned long max_idx;		unsigned long max_idx;
	XA_STATE(xas, &mapping->i_pages, start_pgoff);		XA_STATE(xas, &mapping->i_pages, start_pgoff);
	struct page head, page;		struct page *page;

	rcu_read_lock();		rcu_read_lock();
	xas_for_each(&xas, page, end_pgoff) {		xas_for_each(&xas, page, end_pgoff) {
	@@ -2700,24 +2670,19 @@ void filemap_map_pages(struct vm_fault *vmf,
	if (xa_is_value(page))		if (xa_is_value(page))
	goto next;		goto next;

	head = compound_head(page);

	/*		/*
	* Check for a locked page first, as a speculative		* Check for a locked page first, as a speculative
	* reference may adversely influence page migration.		* reference may adversely influence page migration.
	*/		*/
	if (PageLocked(head))		if (PageLocked(page))
	goto next;		goto next;
	if (!page_cache_get_speculative(head))		if (!page_cache_get_speculative(page))
	goto next;		goto next;

	/* The page was split under us? */		/* Has the page moved or been split? */
	if (compound_head(page) != head)
	goto skip;

	/* Has the page moved? */
	if (unlikely(page != xas_reload(&xas)))		if (unlikely(page != xas_reload(&xas)))
	goto skip;		goto skip;
			page = find_subpage(page, xas.xa_index);

	if (!PageUptodate(page) \|\|		if (!PageUptodate(page) \|\|
	PageReadahead(page) \|\|		PageReadahead(page) \|\|

mm/huge_memory.c

+3 −0

Original line number	Original line	Diff line number	Diff line
	@@ -2496,6 +2496,9 @@ static void __split_huge_page(struct page page, struct list_head list,
	if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))		if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))
	shmem_uncharge(head->mapping->host, 1);		shmem_uncharge(head->mapping->host, 1);
	put_page(head + i);		put_page(head + i);
			} else if (!PageAnon(page)) {
			__xa_store(&head->mapping->i_pages, head[i].index,
			head + i, 0);
	}		}
	}		}

mm/khugepaged.c

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -1374,7 +1374,7 @@ static void collapse_shmem(struct mm_struct *mm,
	result = SCAN_FAIL;		result = SCAN_FAIL;
	goto xa_locked;		goto xa_locked;
	}		}
	xas_store(&xas, new_page + (index % HPAGE_PMD_NR));		xas_store(&xas, new_page);
	nr_none++;		nr_none++;
	continue;		continue;
	}		}
	@@ -1450,7 +1450,7 @@ static void collapse_shmem(struct mm_struct *mm,
	list_add_tail(&page->lru, &pagelist);		list_add_tail(&page->lru, &pagelist);

	/* Finally, replace with the new page. */		/* Finally, replace with the new page. */
	xas_store(&xas, new_page + (index % HPAGE_PMD_NR));		xas_store(&xas, new_page);
	continue;		continue;
	out_unlock:		out_unlock:
	unlock_page(page);		unlock_page(page);

mm/memfd.c

+2 −0

Original line number	Original line	Diff line number	Diff line
	@@ -39,6 +39,7 @@ static void memfd_tag_pins(struct xa_state *xas)
	xas_for_each(xas, page, ULONG_MAX) {		xas_for_each(xas, page, ULONG_MAX) {
	if (xa_is_value(page))		if (xa_is_value(page))
	continue;		continue;
			page = find_subpage(page, xas->xa_index);
	if (page_count(page) - page_mapcount(page) > 1)		if (page_count(page) - page_mapcount(page) > 1)
	xas_set_mark(xas, MEMFD_TAG_PINNED);		xas_set_mark(xas, MEMFD_TAG_PINNED);

	@@ -88,6 +89,7 @@ static int memfd_wait_for_pins(struct address_space *mapping)
	bool clear = true;		bool clear = true;
	if (xa_is_value(page))		if (xa_is_value(page))
	continue;		continue;
			page = find_subpage(page, xas.xa_index);
	if (page_count(page) - page_mapcount(page) != 1) {		if (page_count(page) - page_mapcount(page) != 1) {
	/*		/*
	* On the last scan, we clean up all those tags		* On the last scan, we clean up all those tags

Admin message