hugetlbfs: revert "use i_mmap_rwsem for more pmd sharing synchronization" (ddeaab32) · Commits · 戴 / test

mm/hugetlb.c

+15 −49

Original line number	Diff line number	Diff line
		@@ -3238,7 +3238,6 @@ int copy_hugetlb_page_range(struct mm_struct dst, struct mm_struct src,
		struct page *ptepage;
		unsigned long addr;
		int cow;
		struct address_space *mapping = vma->vm_file->f_mapping;
		struct hstate *h = hstate_vma(vma);
		unsigned long sz = huge_page_size(h);
		struct mmu_notifier_range range;
		@@ -3250,23 +3249,13 @@ int copy_hugetlb_page_range(struct mm_struct dst, struct mm_struct src,
		mmu_notifier_range_init(&range, src, vma->vm_start,
		vma->vm_end);
		mmu_notifier_invalidate_range_start(&range);
		} else {
		/*
		* For shared mappings i_mmap_rwsem must be held to call
		* huge_pte_alloc, otherwise the returned ptep could go
		* away if part of a shared pmd and another thread calls
		* huge_pmd_unshare.
		*/
		i_mmap_lock_read(mapping);
		}

		for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
		spinlock_t src_ptl, dst_ptl;

		src_pte = huge_pte_offset(src, addr, sz);
		if (!src_pte)
		continue;

		dst_pte = huge_pte_alloc(dst, addr, sz);
		if (!dst_pte) {
		ret = -ENOMEM;
		@@ -3337,8 +3326,6 @@ int copy_hugetlb_page_range(struct mm_struct dst, struct mm_struct src,

		if (cow)
		mmu_notifier_invalidate_range_end(&range);
		else
		i_mmap_unlock_read(mapping);

		return ret;
		}
		@@ -3784,18 +3771,14 @@ retry:
		};

		/*
		* hugetlb_fault_mutex and i_mmap_rwsem must be
		* dropped before handling userfault. Reacquire
		* after handling fault to make calling code simpler.
		* hugetlb_fault_mutex must be dropped before
		* handling userfault. Reacquire after handling
		* fault to make calling code simpler.
		*/
		hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping,
		idx, haddr);
		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
		i_mmap_unlock_read(mapping);

		ret = handle_userfault(&vmf, VM_UFFD_MISSING);

		i_mmap_lock_read(mapping);
		mutex_lock(&hugetlb_fault_mutex_table[hash]);
		goto out;
		}
		@@ -3943,11 +3926,6 @@ vm_fault_t hugetlb_fault(struct mm_struct mm, struct vm_area_struct vma,

		ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
		if (ptep) {
		/*
		* Since we hold no locks, ptep could be stale. That is
		* OK as we are only making decisions based on content and
		* not actually modifying content here.
		*/
		entry = huge_ptep_get(ptep);
		if (unlikely(is_hugetlb_entry_migration(entry))) {
		migration_entry_wait_huge(vma, mm, ptep);
		@@ -3955,31 +3933,20 @@ vm_fault_t hugetlb_fault(struct mm_struct mm, struct vm_area_struct vma,
		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
		return VM_FAULT_HWPOISON_LARGE \|
		VM_FAULT_SET_HINDEX(hstate_index(h));
		}

		/*
		* Acquire i_mmap_rwsem before calling huge_pte_alloc and hold
		* until finished with ptep. This prevents huge_pmd_unshare from
		* being called elsewhere and making the ptep no longer valid.
		*
		* ptep could have already be assigned via huge_pte_offset. That
		* is OK, as huge_pte_alloc will return the same value unless
		* something changed.
		*/
		mapping = vma->vm_file->f_mapping;
		i_mmap_lock_read(mapping);
		} else {
		ptep = huge_pte_alloc(mm, haddr, huge_page_size(h));
		if (!ptep) {
		i_mmap_unlock_read(mapping);
		if (!ptep)
		return VM_FAULT_OOM;
		}

		mapping = vma->vm_file->f_mapping;
		idx = vma_hugecache_offset(h, vma, haddr);

		/*
		* Serialize hugepage allocation and instantiation, so that we don't
		* get spurious allocation failures if two CPUs race to instantiate
		* the same page in the page cache.
		*/
		idx = vma_hugecache_offset(h, vma, haddr);
		hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, haddr);
		mutex_lock(&hugetlb_fault_mutex_table[hash]);

		@@ -4067,7 +4034,6 @@ out_ptl:
		}
		out_mutex:
		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
		i_mmap_unlock_read(mapping);
		/*
		* Generally it's safe to hold refcount during waiting page lock. But
		* here we just wait to defer the next page fault to avoid busy loop and
		@@ -4672,12 +4638,10 @@ void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
		* Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
		* and returns the corresponding pte. While this is not necessary for the
		* !shared pmd case because we can allocate the pmd later as well, it makes the
		* code much cleaner.
		*
		* This routine must be called with i_mmap_rwsem held in at least read mode.
		* For hugetlbfs, this prevents removal of any page table entries associated
		* with the address space. This is important as we are setting up sharing
		* based on existing page table entries (mappings).
		* code much cleaner. pmd allocation is essential for the shared case because
		* pud has to be populated inside the same i_mmap_rwsem section - otherwise
		* racing tasks could either miss the sharing (see huge_pte_offset) or select a
		* bad pmd for sharing.
		*/
		pte_t huge_pmd_share(struct mm_struct mm, unsigned long addr, pud_t *pud)
		{
		@@ -4694,6 +4658,7 @@ pte_t huge_pmd_share(struct mm_struct mm, unsigned long addr, pud_t *pud)
		if (!vma_shareable(vma, addr))
		return (pte_t *)pmd_alloc(mm, pud, addr);

		i_mmap_lock_write(mapping);
		vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
		if (svma == vma)
		continue;
		@@ -4723,6 +4688,7 @@ pte_t huge_pmd_share(struct mm_struct mm, unsigned long addr, pud_t *pud)
		spin_unlock(ptl);
		out:
		pte = (pte_t *)pmd_alloc(mm, pud, addr);
		i_mmap_unlock_write(mapping);
		return pte;
		}

		@@ -4733,7 +4699,7 @@ out:
		* indicated by page_count > 1, unmap is achieved by clearing pud and
		* decrementing the ref count. If count == 1, the pte page is not shared.
		*
		* Called with page table lock held and i_mmap_rwsem held in write mode.
		* called with page table lock held.
		*
		* returns: 1 successfully unmapped a shared pte page
		* 0 the underlying pte page is not shared, or it is the last user

mm/memory-failure.c

+2 −14

Original line number	Diff line number	Diff line
		@@ -966,7 +966,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
		enum ttu_flags ttu = TTU_IGNORE_MLOCK \| TTU_IGNORE_ACCESS;
		struct address_space *mapping;
		LIST_HEAD(tokill);
		bool unmap_success = true;
		bool unmap_success;
		int kill = 1, forcekill;
		struct page hpage = hpagep;
		bool mlocked = PageMlocked(hpage);
		@@ -1028,19 +1028,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
		if (kill)
		collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED);

		if (!PageHuge(hpage)) {
		unmap_success = try_to_unmap(hpage, ttu);
		} else if (mapping) {
		/*
		* For hugetlb pages, try_to_unmap could potentially call
		* huge_pmd_unshare. Because of this, take semaphore in
		* write mode here and set TTU_RMAP_LOCKED to indicate we
		* have taken the lock at this higer level.
		*/
		i_mmap_lock_write(mapping);
		unmap_success = try_to_unmap(hpage, ttu\|TTU_RMAP_LOCKED);
		i_mmap_unlock_write(mapping);
		}
		if (!unmap_success)
		pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n",
		pfn, page_mapcount(hpage));

mm/migrate.c

+1 −12

Original line number	Diff line number	Diff line
		@@ -1324,19 +1324,8 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
		goto put_anon;

		if (page_mapped(hpage)) {
		struct address_space *mapping = page_mapping(hpage);

		/*
		* try_to_unmap could potentially call huge_pmd_unshare.
		* Because of this, take semaphore in write mode here and
		* set TTU_RMAP_LOCKED to let lower levels know we have
		* taken the lock.
		*/
		i_mmap_lock_write(mapping);
		try_to_unmap(hpage,
		TTU_MIGRATION\|TTU_IGNORE_MLOCK\|TTU_IGNORE_ACCESS\|
		TTU_RMAP_LOCKED);
		i_mmap_unlock_write(mapping);
		TTU_MIGRATION\|TTU_IGNORE_MLOCK\|TTU_IGNORE_ACCESS);
		page_was_mapped = 1;
		}

mm/rmap.c

+0 −4

Original line number	Diff line number	Diff line
		@@ -25,7 +25,6 @@
		* page->flags PG_locked (lock_page)
		* hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share)
		* mapping->i_mmap_rwsem
		* hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
		* anon_vma->rwsem
		* mm->page_table_lock or pte_lock
		* zone_lru_lock (in mark_page_accessed, isolate_lru_page)
		@@ -1379,9 +1378,6 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma,
		/*
		* If sharing is possible, start and end will be adjusted
		* accordingly.
		*
		* If called for a huge page, caller must hold i_mmap_rwsem
		* in write mode as it is possible to call huge_pmd_unshare.
		*/
		adjust_range_if_pmd_sharing_possible(vma, &range.start,
		&range.end);

mm/userfaultfd.c

+2 −9

Original line number	Diff line number	Diff line
		@@ -267,14 +267,10 @@ retry:
		VM_BUG_ON(dst_addr & ~huge_page_mask(h));

		/*
		* Serialize via i_mmap_rwsem and hugetlb_fault_mutex.
		* i_mmap_rwsem ensures the dst_pte remains valid even
		* in the case of shared pmds. fault mutex prevents
		* races with other faulting threads.
		* Serialize via hugetlb_fault_mutex
		*/
		mapping = dst_vma->vm_file->f_mapping;
		i_mmap_lock_read(mapping);
		idx = linear_page_index(dst_vma, dst_addr);
		mapping = dst_vma->vm_file->f_mapping;
		hash = hugetlb_fault_mutex_hash(h, dst_mm, dst_vma, mapping,
		idx, dst_addr);
		mutex_lock(&hugetlb_fault_mutex_table[hash]);
		@@ -283,7 +279,6 @@ retry:
		dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h));
		if (!dst_pte) {
		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
		i_mmap_unlock_read(mapping);
		goto out_unlock;
		}

		@@ -291,7 +286,6 @@ retry:
		dst_pteval = huge_ptep_get(dst_pte);
		if (!huge_pte_none(dst_pteval)) {
		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
		i_mmap_unlock_read(mapping);
		goto out_unlock;
		}

		@@ -299,7 +293,6 @@ retry:
		dst_addr, src_addr, &page);

		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
		i_mmap_unlock_read(mapping);
		vm_alloc_shared = vm_shared;

		cond_resched();

Admin message