x86, pat: Use page flags to track memtypes of RAM pages (f5841740) · Commits · 戴 / test

arch/x86/include/asm/cacheflush.h

+52 −2

Original line number	Diff line number	Diff line
		@@ -43,8 +43,58 @@ static inline void copy_from_user_page(struct vm_area_struct *vma,
		memcpy(dst, src, len);
		}

		#define PG_non_WB PG_arch_1
		PAGEFLAG(NonWB, non_WB)
		#define PG_WC PG_arch_1
		PAGEFLAG(WC, WC)

		#ifdef CONFIG_X86_PAT
		/*
		* X86 PAT uses page flags WC and Uncached together to keep track of
		* memory type of pages that have backing page struct. X86 PAT supports 3
		* different memory types, _PAGE_CACHE_WB, _PAGE_CACHE_WC and
		* _PAGE_CACHE_UC_MINUS and fourth state where page's memory type has not
		* been changed from its default (value of -1 used to denote this).
		* Note we do not support _PAGE_CACHE_UC here.
		*
		* Caller must hold memtype_lock for atomicity.
		*/
		static inline unsigned long get_page_memtype(struct page *pg)
		{
		if (!PageUncached(pg) && !PageWC(pg))
		return -1;
		else if (!PageUncached(pg) && PageWC(pg))
		return _PAGE_CACHE_WC;
		else if (PageUncached(pg) && !PageWC(pg))
		return _PAGE_CACHE_UC_MINUS;
		else
		return _PAGE_CACHE_WB;
		}

		static inline void set_page_memtype(struct page *pg, unsigned long memtype)
		{
		switch (memtype) {
		case _PAGE_CACHE_WC:
		ClearPageUncached(pg);
		SetPageWC(pg);
		break;
		case _PAGE_CACHE_UC_MINUS:
		SetPageUncached(pg);
		ClearPageWC(pg);
		break;
		case _PAGE_CACHE_WB:
		SetPageUncached(pg);
		SetPageWC(pg);
		break;
		default:
		case -1:
		ClearPageUncached(pg);
		ClearPageWC(pg);
		break;
		}
		}
		#else
		static inline unsigned long get_page_memtype(struct page *pg) { return -1; }
		static inline void set_page_memtype(struct page *pg, unsigned long memtype) { }
		#endif

		/*
		* The set_memory_* API can be used to change various attributes of a virtual

arch/x86/mm/pat.c

+50 −41

Original line number	Diff line number	Diff line
		@@ -288,63 +288,61 @@ static int pat_pagerange_is_ram(unsigned long start, unsigned long end)
		}

		/*
		* For RAM pages, mark the pages as non WB memory type using
		* PageNonWB (PG_arch_1). We allow only one set_memory_uc() or
		* set_memory_wc() on a RAM page at a time before marking it as WB again.
		* This is ok, because only one driver will be owning the page and
		* doing set_memory_*() calls.
		* For RAM pages, we use page flags to mark the pages with appropriate type.
		* Here we do two pass:
		* - Find the memtype of all the pages in the range, look for any conflicts
		* - In case of no conflicts, set the new memtype for pages in the range
		*
		* For now, we use PageNonWB to track that the RAM page is being mapped
		* as non WB. In future, we will have to use one more flag
		* (or some other mechanism in page_struct) to distinguish between
		* UC and WC mapping.
		* Caller must hold memtype_lock for atomicity.
		*/
		static int reserve_ram_pages_type(u64 start, u64 end, unsigned long req_type,
		unsigned long *new_type)
		{
		struct page *page;
		u64 pfn, end_pfn;
		u64 pfn;

		if (req_type == _PAGE_CACHE_UC) {
		/* We do not support strong UC */
		WARN_ON_ONCE(1);
		req_type = _PAGE_CACHE_UC_MINUS;
		}

		for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
		unsigned long type;

		page = pfn_to_page(pfn);
		if (page_mapped(page) \|\| PageNonWB(page))
		goto out;
		type = get_page_memtype(page);
		if (type != -1) {
		printk(KERN_INFO "reserve_ram_pages_type failed "
		"0x%Lx-0x%Lx, track 0x%lx, req 0x%lx\n",
		start, end, type, req_type);
		if (new_type)
		*new_type = type;

		SetPageNonWB(page);
		return -EBUSY;
		}
		return 0;
		}

		if (new_type)
		*new_type = req_type;

		out:
		end_pfn = pfn;
		for (pfn = (start >> PAGE_SHIFT); pfn < end_pfn; ++pfn) {
		for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
		page = pfn_to_page(pfn);
		ClearPageNonWB(page);
		set_page_memtype(page, req_type);
		}

		return -EINVAL;
		return 0;
		}

		static int free_ram_pages_type(u64 start, u64 end)
		{
		struct page *page;
		u64 pfn, end_pfn;
		u64 pfn;

		for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
		page = pfn_to_page(pfn);
		if (page_mapped(page) \|\| !PageNonWB(page))
		goto out;

		ClearPageNonWB(page);
		set_page_memtype(page, -1);
		}
		return 0;

		out:
		end_pfn = pfn;
		for (pfn = (start >> PAGE_SHIFT); pfn < end_pfn; ++pfn) {
		page = pfn_to_page(pfn);
		SetPageNonWB(page);
		}
		return -EINVAL;
		}

		/*
		@@ -405,11 +403,16 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
		*new_type = actual_type;

		is_range_ram = pat_pagerange_is_ram(start, end);
		if (is_range_ram == 1)
		return reserve_ram_pages_type(start, end, req_type,
		new_type);
		else if (is_range_ram < 0)
		if (is_range_ram == 1) {

		spin_lock(&memtype_lock);
		err = reserve_ram_pages_type(start, end, req_type, new_type);
		spin_unlock(&memtype_lock);

		return err;
		} else if (is_range_ram < 0) {
		return -EINVAL;
		}

		new = kmalloc(sizeof(struct memtype), GFP_KERNEL);
		if (!new)
		@@ -505,10 +508,16 @@ int free_memtype(u64 start, u64 end)
		return 0;

		is_range_ram = pat_pagerange_is_ram(start, end);
		if (is_range_ram == 1)
		return free_ram_pages_type(start, end);
		else if (is_range_ram < 0)
		if (is_range_ram == 1) {

		spin_lock(&memtype_lock);
		err = free_ram_pages_type(start, end);
		spin_unlock(&memtype_lock);

		return err;
		} else if (is_range_ram < 0) {
		return -EINVAL;
		}

		spin_lock(&memtype_lock);

Admin message