hugetlbfs: fix anon huge page migration race (336bf30e) · Commits · 戴 / test

mm/hugetlb.c

+5 −85

Original line number	Diff line number	Diff line
		@@ -1567,106 +1567,26 @@ int PageHeadHuge(struct page *page_head)
		return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR;
		}

		/*
		* Find address_space associated with hugetlbfs page.
		* Upon entry page is locked and page 'was' mapped although mapped state
		* could change. If necessary, use anon_vma to find vma and associated
		* address space. The returned mapping may be stale, but it can not be
		* invalid as page lock (which is held) is required to destroy mapping.
		*/
		static struct address_space _get_hugetlb_page_mapping(struct page hpage)
		{
		struct anon_vma *anon_vma;
		pgoff_t pgoff_start, pgoff_end;
		struct anon_vma_chain *avc;
		struct address_space *mapping = page_mapping(hpage);

		/* Simple file based mapping */
		if (mapping)
		return mapping;

		/*
		* Even anonymous hugetlbfs mappings are associated with an
		* underlying hugetlbfs file (see hugetlb_file_setup in mmap
		* code). Find a vma associated with the anonymous vma, and
		* use the file pointer to get address_space.
		*/
		anon_vma = page_lock_anon_vma_read(hpage);
		if (!anon_vma)
		return mapping; /* NULL */

		/* Use first found vma */
		pgoff_start = page_to_pgoff(hpage);
		pgoff_end = pgoff_start + pages_per_huge_page(page_hstate(hpage)) - 1;
		anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
		pgoff_start, pgoff_end) {
		struct vm_area_struct *vma = avc->vma;

		mapping = vma->vm_file->f_mapping;
		break;
		}

		anon_vma_unlock_read(anon_vma);
		return mapping;
		}

		/*
		* Find and lock address space (mapping) in write mode.
		*
		* Upon entry, the page is locked which allows us to find the mapping
		* even in the case of an anon page. However, locking order dictates
		* the i_mmap_rwsem be acquired BEFORE the page lock. This is hugetlbfs
		* specific. So, we first try to lock the sema while still holding the
		* page lock. If this works, great! If not, then we need to drop the
		* page lock and then acquire i_mmap_rwsem and reacquire page lock. Of
		* course, need to revalidate state along the way.
		* Upon entry, the page is locked which means that page_mapping() is
		* stable. Due to locking order, we can only trylock_write. If we can
		* not get the lock, simply return NULL to caller.
		*/
		struct address_space hugetlb_page_mapping_lock_write(struct page hpage)
		{
		struct address_space mapping, mapping2;
		struct address_space *mapping = page_mapping(hpage);

		mapping = _get_hugetlb_page_mapping(hpage);
		retry:
		if (!mapping)
		return mapping;

		/*
		* If no contention, take lock and return
		*/
		if (i_mmap_trylock_write(mapping))
		return mapping;

		/*
		* Must drop page lock and wait on mapping sema.
		* Note: Once page lock is dropped, mapping could become invalid.
		* As a hack, increase map count until we lock page again.
		*/
		atomic_inc(&hpage->_mapcount);
		unlock_page(hpage);
		i_mmap_lock_write(mapping);
		lock_page(hpage);
		atomic_add_negative(-1, &hpage->_mapcount);

		/* verify page is still mapped */
		if (!page_mapped(hpage)) {
		i_mmap_unlock_write(mapping);
		return NULL;
		}

		/*
		* Get address space again and verify it is the same one
		* we locked. If not, drop lock and retry.
		*/
		mapping2 = _get_hugetlb_page_mapping(hpage);
		if (mapping2 != mapping) {
		i_mmap_unlock_write(mapping);
		mapping = mapping2;
		goto retry;
		}

		return mapping;
		}

		pgoff_t __basepage_index(struct page *page)
		{
		struct page *page_head = compound_head(page);

mm/memory-failure.c

+17 −19

Original line number	Diff line number	Diff line
		@@ -1057,28 +1057,26 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
		if (!PageHuge(hpage)) {
		unmap_success = try_to_unmap(hpage, ttu);
		} else {
		if (!PageAnon(hpage)) {
		/*
		* For hugetlb pages, try_to_unmap could potentially call
		* huge_pmd_unshare. Because of this, take semaphore in
		* write mode here and set TTU_RMAP_LOCKED to indicate we
		* have taken the lock at this higer level.
		*
		* Note that the call to hugetlb_page_mapping_lock_write
		* is necessary even if mapping is already set. It handles
		* ugliness of potentially having to drop page lock to obtain
		* i_mmap_rwsem.
		* For hugetlb pages in shared mappings, try_to_unmap
		* could potentially call huge_pmd_unshare. Because of
		* this, take semaphore in write mode here and set
		* TTU_RMAP_LOCKED to indicate we have taken the lock
		* at this higer level.
		*/
		mapping = hugetlb_page_mapping_lock_write(hpage);

		if (mapping) {
		unmap_success = try_to_unmap(hpage,
		ttu\|TTU_RMAP_LOCKED);
		i_mmap_unlock_write(mapping);
		} else {
		pr_info("Memory failure: %#lx: could not find mapping for mapped huge page\n",
		pfn);
		pr_info("Memory failure: %#lx: could not lock mapping for mapped huge page\n", pfn);
		unmap_success = false;
		}
		} else {
		unmap_success = try_to_unmap(hpage, ttu);
		}
		}
		if (!unmap_success)
		pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n",

mm/migrate.c

+24 −20

Original line number	Diff line number	Diff line
		@@ -1328,34 +1328,38 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
		goto put_anon;

		if (page_mapped(hpage)) {
		bool mapping_locked = false;
		enum ttu_flags ttu = TTU_MIGRATION\|TTU_IGNORE_MLOCK\|
		TTU_IGNORE_ACCESS;

		if (!PageAnon(hpage)) {
		/*
		* try_to_unmap could potentially call huge_pmd_unshare.
		* Because of this, take semaphore in write mode here and
		* set TTU_RMAP_LOCKED to let lower levels know we have
		* taken the lock.
		* In shared mappings, try_to_unmap could potentially
		* call huge_pmd_unshare. Because of this, take
		* semaphore in write mode here and set TTU_RMAP_LOCKED
		* to let lower levels know we have taken the lock.
		*/
		mapping = hugetlb_page_mapping_lock_write(hpage);
		if (unlikely(!mapping))
		goto unlock_put_anon;

		try_to_unmap(hpage,
		TTU_MIGRATION\|TTU_IGNORE_MLOCK\|TTU_IGNORE_ACCESS\|
		TTU_RMAP_LOCKED);
		mapping_locked = true;
		ttu \|= TTU_RMAP_LOCKED;
		}

		try_to_unmap(hpage, ttu);
		page_was_mapped = 1;
		/*
		* Leave mapping locked until after subsequent call to
		* remove_migration_ptes()
		*/

		if (mapping_locked)
		i_mmap_unlock_write(mapping);
		}

		if (!page_mapped(hpage))
		rc = move_to_new_page(new_hpage, hpage, mode);

		if (page_was_mapped) {
		if (page_was_mapped)
		remove_migration_ptes(hpage,
		rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, true);
		i_mmap_unlock_write(mapping);
		}
		rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);

		unlock_put_anon:
		unlock_page(new_hpage);

mm/rmap.c

+1 −4

Original line number	Diff line number	Diff line
		@@ -1413,9 +1413,6 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma,
		/*
		* If sharing is possible, start and end will be adjusted
		* accordingly.
		*
		* If called for a huge page, caller must hold i_mmap_rwsem
		* in write mode as it is possible to call huge_pmd_unshare.
		*/
		adjust_range_if_pmd_sharing_possible(vma, &range.start,
		&range.end);
		@@ -1462,7 +1459,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma,
		subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
		address = pvmw.address;

		if (PageHuge(page)) {
		if (PageHuge(page) && !PageAnon(page)) {
		/*
		* To call huge_pmd_unshare, i_mmap_rwsem must be
		* held in write mode. Caller needs to explicitly

Admin message