mm: pagewalk: add p4d_entry() and pgd_entry() (3afc4236) · Commits · 戴 / test

include/linux/pagewalk.h

+28 −6

Original line number	Original line	Diff line number	Diff line
	@@ -8,15 +8,15 @@ struct mm_walk;

	/**		/**
	* mm_walk_ops - callbacks for walk_page_range		* mm_walk_ops - callbacks for walk_page_range
	* @pud_entry: if set, called for each non-empty PUD (2nd-level) entry		* @pgd_entry: if set, called for each non-empty PGD (top-level) entry
	* this handler should only handle pud_trans_huge() puds.		* @p4d_entry: if set, called for each non-empty P4D entry
	* the pmd_entry or pte_entry callbacks will be used for		* @pud_entry: if set, called for each non-empty PUD entry
	* regular PUDs.		* @pmd_entry: if set, called for each non-empty PMD entry
	* @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
	* this handler is required to be able to handle		* this handler is required to be able to handle
	* pmd_trans_huge() pmds. They may simply choose to		* pmd_trans_huge() pmds. They may simply choose to
	* split_huge_page() instead of handling it explicitly.		* split_huge_page() instead of handling it explicitly.
	* @pte_entry: if set, called for each non-empty PTE (4th-level) entry		* @pte_entry: if set, called for each non-empty PTE (lowest-level)
			* entry
	* @pte_hole: if set, called for each hole at all levels		* @pte_hole: if set, called for each hole at all levels
	* @hugetlb_entry: if set, called for each hugetlb entry		* @hugetlb_entry: if set, called for each hugetlb entry
	* @test_walk: caller specific callback function to determine whether		* @test_walk: caller specific callback function to determine whether
	@@ -27,8 +27,15 @@ struct mm_walk;
	* @pre_vma: if set, called before starting walk on a non-null vma.		* @pre_vma: if set, called before starting walk on a non-null vma.
	* @post_vma: if set, called after a walk on a non-null vma, provided		* @post_vma: if set, called after a walk on a non-null vma, provided
	* that @pre_vma and the vma walk succeeded.		* that @pre_vma and the vma walk succeeded.
			*
			* p?d_entry callbacks are called even if those levels are folded on a
			* particular architecture/configuration.
	*/		*/
	struct mm_walk_ops {		struct mm_walk_ops {
			int (pgd_entry)(pgd_t pgd, unsigned long addr,
			unsigned long next, struct mm_walk *walk);
			int (p4d_entry)(p4d_t p4d, unsigned long addr,
			unsigned long next, struct mm_walk *walk);
	int (pud_entry)(pud_t pud, unsigned long addr,		int (pud_entry)(pud_t pud, unsigned long addr,
	unsigned long next, struct mm_walk *walk);		unsigned long next, struct mm_walk *walk);
	int (pmd_entry)(pmd_t pmd, unsigned long addr,		int (pmd_entry)(pmd_t pmd, unsigned long addr,
	@@ -47,11 +54,25 @@ struct mm_walk_ops {
	void (post_vma)(struct mm_walk walk);		void (post_vma)(struct mm_walk walk);
	};		};

			/*
			* Action for pud_entry / pmd_entry callbacks.
			* ACTION_SUBTREE is the default
			*/
			enum page_walk_action {
			/* Descend to next level, splitting huge pages if needed and possible */
			ACTION_SUBTREE = 0,
			/* Continue to next entry at this level (ignoring any subtree) */
			ACTION_CONTINUE = 1,
			/* Call again for this entry */
			ACTION_AGAIN = 2
			};

	/**		/**
	* mm_walk - walk_page_range data		* mm_walk - walk_page_range data
	* @ops: operation to call during the walk		* @ops: operation to call during the walk
	* @mm: mm_struct representing the target process of page table walk		* @mm: mm_struct representing the target process of page table walk
	* @vma: vma currently walked (NULL if walking outside vmas)		* @vma: vma currently walked (NULL if walking outside vmas)
			* @action: next action to perform (see enum page_walk_action)
	* @private: private data for callbacks' usage		* @private: private data for callbacks' usage
	*		*
	* (see the comment on walk_page_range() for more details)		* (see the comment on walk_page_range() for more details)
	@@ -60,6 +81,7 @@ struct mm_walk {
	const struct mm_walk_ops *ops;		const struct mm_walk_ops *ops;
	struct mm_struct *mm;		struct mm_struct *mm;
	struct vm_area_struct *vma;		struct vm_area_struct *vma;
			enum page_walk_action action;
	void *private;		void *private;
	};		};

mm/hmm.c

+32 −26

Original line number	Original line	Diff line number	Diff line
	@@ -474,23 +474,32 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
	{		{
	struct hmm_vma_walk *hmm_vma_walk = walk->private;		struct hmm_vma_walk *hmm_vma_walk = walk->private;
	struct hmm_range *range = hmm_vma_walk->range;		struct hmm_range *range = hmm_vma_walk->range;
	unsigned long addr = start, next;		unsigned long addr = start;
	pmd_t *pmdp;
	pud_t pud;		pud_t pud;
	int ret;		int ret = 0;
			spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma);

			if (!ptl)
			return 0;

			/* Normally we don't want to split the huge page */
			walk->action = ACTION_CONTINUE;

	again:
	pud = READ_ONCE(*pudp);		pud = READ_ONCE(*pudp);
	if (pud_none(pud))		if (pud_none(pud)) {
	return hmm_vma_walk_hole(start, end, walk);		ret = hmm_vma_walk_hole(start, end, walk);
			goto out_unlock;
			}

	if (pud_huge(pud) && pud_devmap(pud)) {		if (pud_huge(pud) && pud_devmap(pud)) {
	unsigned long i, npages, pfn;		unsigned long i, npages, pfn;
	uint64_t *pfns, cpu_flags;		uint64_t *pfns, cpu_flags;
	bool fault, write_fault;		bool fault, write_fault;

	if (!pud_present(pud))		if (!pud_present(pud)) {
	return hmm_vma_walk_hole(start, end, walk);		ret = hmm_vma_walk_hole(start, end, walk);
			goto out_unlock;
			}

	i = (addr - range->start) >> PAGE_SHIFT;		i = (addr - range->start) >> PAGE_SHIFT;
	npages = (end - addr) >> PAGE_SHIFT;		npages = (end - addr) >> PAGE_SHIFT;
	@@ -499,16 +508,20 @@ again:
	cpu_flags = pud_to_hmm_pfn_flags(range, pud);		cpu_flags = pud_to_hmm_pfn_flags(range, pud);
	hmm_range_need_fault(hmm_vma_walk, pfns, npages,		hmm_range_need_fault(hmm_vma_walk, pfns, npages,
	cpu_flags, &fault, &write_fault);		cpu_flags, &fault, &write_fault);
	if (fault \|\| write_fault)		if (fault \|\| write_fault) {
	return hmm_vma_walk_hole_(addr, end, fault,		ret = hmm_vma_walk_hole_(addr, end, fault,
	write_fault, walk);		write_fault, walk);
			goto out_unlock;
			}

	pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);		pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
	for (i = 0; i < npages; ++i, ++pfn) {		for (i = 0; i < npages; ++i, ++pfn) {
	hmm_vma_walk->pgmap = get_dev_pagemap(pfn,		hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
	hmm_vma_walk->pgmap);		hmm_vma_walk->pgmap);
	if (unlikely(!hmm_vma_walk->pgmap))		if (unlikely(!hmm_vma_walk->pgmap)) {
	return -EBUSY;		ret = -EBUSY;
			goto out_unlock;
			}
	pfns[i] = hmm_device_entry_from_pfn(range, pfn) \|		pfns[i] = hmm_device_entry_from_pfn(range, pfn) \|
	cpu_flags;		cpu_flags;
	}		}
	@@ -517,22 +530,15 @@ again:
	hmm_vma_walk->pgmap = NULL;		hmm_vma_walk->pgmap = NULL;
	}		}
	hmm_vma_walk->last = end;		hmm_vma_walk->last = end;
	return 0;		goto out_unlock;
	}		}

	split_huge_pud(walk->vma, pudp, addr);		/* Ask for the PUD to be split */
	if (pud_none(*pudp))		walk->action = ACTION_SUBTREE;
	goto again;

	pmdp = pmd_offset(pudp, addr);		out_unlock:
	do {		spin_unlock(ptl);
	next = pmd_addr_end(addr, end);
	ret = hmm_vma_walk_pmd(pmdp, addr, next, walk);
	if (ret)
	return ret;		return ret;
	} while (pmdp++, addr = next, addr != end);

	return 0;
	}		}
	#else		#else
	#define hmm_vma_walk_pud NULL		#define hmm_vma_walk_pud NULL

mm/pagewalk.c

+35 −15

Original line number	Original line	Diff line number	Diff line
	@@ -46,6 +46,9 @@ again:
	break;		break;
	continue;		continue;
	}		}

			walk->action = ACTION_SUBTREE;

	/*		/*
	* This implies that each ->pmd_entry() handler		* This implies that each ->pmd_entry() handler
	* needs to know about pmd_trans_huge() pmds		* needs to know about pmd_trans_huge() pmds
	@@ -55,16 +58,21 @@ again:
	if (err)		if (err)
	break;		break;

			if (walk->action == ACTION_AGAIN)
			goto again;

	/*		/*
	* Check this here so we only break down trans_huge		* Check this here so we only break down trans_huge
	* pages when we _need_ to		* pages when we _need_ to
	*/		*/
	if (!ops->pte_entry)		if (walk->action == ACTION_CONTINUE \|\|
			!(ops->pte_entry))
	continue;		continue;

	split_huge_pmd(walk->vma, pmd, addr);		split_huge_pmd(walk->vma, pmd, addr);
	if (pmd_trans_unstable(pmd))		if (pmd_trans_unstable(pmd))
	goto again;		goto again;

	err = walk_pte_range(pmd, addr, next, walk);		err = walk_pte_range(pmd, addr, next, walk);
	if (err)		if (err)
	break;		break;
	@@ -93,23 +101,24 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
	continue;		continue;
	}		}

	if (ops->pud_entry) {		walk->action = ACTION_SUBTREE;
	spinlock_t *ptl = pud_trans_huge_lock(pud, walk->vma);

	if (ptl) {		if (ops->pud_entry)
	err = ops->pud_entry(pud, addr, next, walk);		err = ops->pud_entry(pud, addr, next, walk);
	spin_unlock(ptl);
	if (err)		if (err)
	break;		break;

			if (walk->action == ACTION_AGAIN)
			goto again;

			if (walk->action == ACTION_CONTINUE \|\|
			!(ops->pmd_entry \|\| ops->pte_entry))
	continue;		continue;
	}
	}

	split_huge_pud(walk->vma, pud, addr);		split_huge_pud(walk->vma, pud, addr);
	if (pud_none(*pud))		if (pud_none(*pud))
	goto again;		goto again;

	if (ops->pmd_entry \|\| ops->pte_entry)
	err = walk_pmd_range(pud, addr, next, walk);		err = walk_pmd_range(pud, addr, next, walk);
	if (err)		if (err)
	break;		break;
	@@ -136,7 +145,12 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
	break;		break;
	continue;		continue;
	}		}
	if (ops->pmd_entry \|\| ops->pte_entry)		if (ops->p4d_entry) {
			err = ops->p4d_entry(p4d, addr, next, walk);
			if (err)
			break;
			}
			if (ops->pud_entry \|\| ops->pmd_entry \|\| ops->pte_entry)
	err = walk_pud_range(p4d, addr, next, walk);		err = walk_pud_range(p4d, addr, next, walk);
	if (err)		if (err)
	break;		break;
	@@ -163,7 +177,13 @@ static int walk_pgd_range(unsigned long addr, unsigned long end,
	break;		break;
	continue;		continue;
	}		}
	if (ops->pmd_entry \|\| ops->pte_entry)		if (ops->pgd_entry) {
			err = ops->pgd_entry(pgd, addr, next, walk);
			if (err)
			break;
			}
			if (ops->p4d_entry \|\| ops->pud_entry \|\| ops->pmd_entry \|\|
			ops->pte_entry)
	err = walk_p4d_range(pgd, addr, next, walk);		err = walk_p4d_range(pgd, addr, next, walk);
	if (err)		if (err)
	break;		break;

Admin message