Commit 3b50a6e5 authored by Ralph Campbell's avatar Ralph Campbell Committed by Jason Gunthorpe
Browse files

mm/hmm: provide the page mapping order in hmm_range_fault() 

hmm_range_fault() returns an array of page frame numbers and flags for how
the pages are mapped in the requested process' page tables. The PFN can be
used to get the struct page with hmm_pfn_to_page() and the page size order
can be determined with compound_order(page).

However, if the page is larger than order 0 (PAGE_SIZE), there is no
indication that a compound page is mapped by the CPU using a larger page
size. Without this information, the caller can't safely use a large device
PTE to map the compound page because the CPU might be using smaller PTEs
with different read/write permissions.

Add a new function hmm_pfn_to_map_order() to return the mapping size order
so that callers know the pages are being mapped with consistent
permissions and a large device page table mapping can be used if one is
available.

This will allow devices to optimize mapping the page into HW by avoiding
or batching work for huge pages. For instance the dma_map can be done with
a high order directly.

Link: https://lore.kernel.org/r/20200701225352.9649-3-rcampbell@nvidia.com


Signed-off-by: default avatarRalph Campbell <rcampbell@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent dcb7fd82

include/linux/hmm.h

+22 −2
Original line number Diff line number Diff line
@@ -37,16 +37,17 @@ 
 *                     will fail. Must be combined with HMM_PFN_REQ_FAULT.

 */

enum hmm_pfn_flags {

	/* Output flags */

	/* Output fields and flags */

	HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),

	HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),

	HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),

	HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),



	/* Input flags */

	HMM_PFN_REQ_FAULT = HMM_PFN_VALID,

	HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,



	HMM_PFN_FLAGS = HMM_PFN_VALID | HMM_PFN_WRITE | HMM_PFN_ERROR,

	HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,

};



/*
@@ -61,6 +62,25 @@ static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn) 
	return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);

}



/*

 * hmm_pfn_to_map_order() - return the CPU mapping size order

 *

 * This is optionally useful to optimize processing of the pfn result

 * array. It indicates that the page starts at the order aligned VA and is

 * 1<<order bytes long.  Every pfn within an high order page will have the

 * same pfn flags, both access protections and the map_order.  The caller must

 * be careful with edge cases as the start and end VA of the given page may

 * extend past the range used with hmm_range_fault().

 *

 * This must be called under the caller 'user_lock' after a successful

 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID

 * already.

 */

static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)

{

	return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;

}



/*

 * struct hmm_range - track invalidation lock on virtual address range

 *

mm/hmm.c

+13 −3
Original line number Diff line number Diff line
@@ -165,12 +165,19 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, 
	return hmm_pfns_fill(addr, end, range, 0);

}



static inline unsigned long hmm_pfn_flags_order(unsigned long order)

{

	return order << HMM_PFN_ORDER_SHIFT;

}



static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,

						 pmd_t pmd)

{

	if (pmd_protnone(pmd))

		return 0;

	return pmd_write(pmd) ? (HMM_PFN_VALID | HMM_PFN_WRITE) : HMM_PFN_VALID;

	return (pmd_write(pmd) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :

				 HMM_PFN_VALID) |

	       hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT);

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -389,7 +396,9 @@ static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range *range, 
{

	if (!pud_present(pud))

		return 0;

	return pud_write(pud) ? (HMM_PFN_VALID | HMM_PFN_WRITE) : HMM_PFN_VALID;

	return (pud_write(pud) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :

				 HMM_PFN_VALID) |

	       hmm_pfn_flags_order(PUD_SHIFT - PAGE_SHIFT);

}



static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
@@ -474,7 +483,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, 


	i = (start - range->start) >> PAGE_SHIFT;

	pfn_req_flags = range->hmm_pfns[i];

	cpu_flags = pte_to_hmm_pfn_flags(range, entry);

	cpu_flags = pte_to_hmm_pfn_flags(range, entry) |

		    hmm_pfn_flags_order(huge_page_order(hstate_vma(vma)));

	required_fault =

		hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);

	if (required_fault) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE