Commit 4f646254 authored by Pauli Nieminen's avatar Pauli Nieminen Committed by Dave Airlie
Browse files

arch/x86: Add array variants for setting memory to wc caching. 



Setting single memory pages at a time to wc takes a lot time in cache flush. To
reduce number of cache flush set_pages_array_wc and set_memory_array_wc can be
used to set multiple pages to WC with single cache flush.

This improves allocation performance for wc cached pages in drm/ttm.

CC: Suresh Siddha <suresh.b.siddha@intel.com>
CC: Venkatesh Pallipadi <venkatesh.pallipadi@gmail.com>
Signed-off-by: default avatarPauli Nieminen <suokkos@gmail.com>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent bf62acde

arch/x86/include/asm/cacheflush.h

+2 −0
Original line number Diff line number Diff line
@@ -139,9 +139,11 @@ int set_memory_np(unsigned long addr, int numpages); 
int set_memory_4k(unsigned long addr, int numpages);



int set_memory_array_uc(unsigned long *addr, int addrinarray);

int set_memory_array_wc(unsigned long *addr, int addrinarray);

int set_memory_array_wb(unsigned long *addr, int addrinarray);



int set_pages_array_uc(struct page **pages, int addrinarray);

int set_pages_array_wc(struct page **pages, int addrinarray);

int set_pages_array_wb(struct page **pages, int addrinarray);



/*

arch/x86/mm/pageattr.c

+45 −8
Original line number Diff line number Diff line
@@ -997,7 +997,8 @@ out_err: 
}

EXPORT_SYMBOL(set_memory_uc);



int set_memory_array_uc(unsigned long *addr, int addrinarray)

int _set_memory_array(unsigned long *addr, int addrinarray,

		unsigned long new_type)

{

	int i, j;

	int ret;
@@ -1007,13 +1008,19 @@ int set_memory_array_uc(unsigned long *addr, int addrinarray) 
	 */

	for (i = 0; i < addrinarray; i++) {

		ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,

					_PAGE_CACHE_UC_MINUS, NULL);

					new_type, NULL);

		if (ret)

			goto out_free;

	}



	ret = change_page_attr_set(addr, addrinarray,

				    __pgprot(_PAGE_CACHE_UC_MINUS), 1);



	if (!ret && new_type == _PAGE_CACHE_WC)

		ret = change_page_attr_set_clr(addr, addrinarray,

					       __pgprot(_PAGE_CACHE_WC),

					       __pgprot(_PAGE_CACHE_MASK),

					       0, CPA_ARRAY, NULL);

	if (ret)

		goto out_free;
@@ -1025,8 +1032,19 @@ out_free: 


	return ret;

}



int set_memory_array_uc(unsigned long *addr, int addrinarray)

{

	return _set_memory_array(addr, addrinarray, _PAGE_CACHE_UC_MINUS);

}

EXPORT_SYMBOL(set_memory_array_uc);



int set_memory_array_wc(unsigned long *addr, int addrinarray)

{

	return _set_memory_array(addr, addrinarray, _PAGE_CACHE_WC);

}

EXPORT_SYMBOL(set_memory_array_wc);



int _set_memory_wc(unsigned long addr, int numpages)

{

	int ret;
@@ -1153,26 +1171,34 @@ int set_pages_uc(struct page *page, int numpages) 
}

EXPORT_SYMBOL(set_pages_uc);



int set_pages_array_uc(struct page **pages, int addrinarray)

static int _set_pages_array(struct page **pages, int addrinarray,

		unsigned long new_type)

{

	unsigned long start;

	unsigned long end;

	int i;

	int free_idx;

	int ret;



	for (i = 0; i < addrinarray; i++) {

		if (PageHighMem(pages[i]))

			continue;

		start = page_to_pfn(pages[i]) << PAGE_SHIFT;

		end = start + PAGE_SIZE;

		if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL))

		if (reserve_memtype(start, end, new_type, NULL))

			goto err_out;

	}



	if (cpa_set_pages_array(pages, addrinarray,

			__pgprot(_PAGE_CACHE_UC_MINUS)) == 0) {

	ret = cpa_set_pages_array(pages, addrinarray,

			__pgprot(_PAGE_CACHE_UC_MINUS));

	if (!ret && new_type == _PAGE_CACHE_WC)

		ret = change_page_attr_set_clr(NULL, addrinarray,

					       __pgprot(_PAGE_CACHE_WC),

					       __pgprot(_PAGE_CACHE_MASK),

					       0, CPA_PAGES_ARRAY, pages);

	if (ret)

		goto err_out;

	return 0; /* Success */

	}

err_out:

	free_idx = i;

	for (i = 0; i < free_idx; i++) {
@@ -1184,8 +1210,19 @@ err_out: 
	}

	return -EINVAL;

}



int set_pages_array_uc(struct page **pages, int addrinarray)

{

	return _set_pages_array(pages, addrinarray, _PAGE_CACHE_UC_MINUS);

}

EXPORT_SYMBOL(set_pages_array_uc);



int set_pages_array_wc(struct page **pages, int addrinarray)

{

	return _set_pages_array(pages, addrinarray, _PAGE_CACHE_WC);

}

EXPORT_SYMBOL(set_pages_array_wc);



int set_pages_wb(struct page *page, int numpages)

{

	unsigned long addr = (unsigned long)page_address(page);

CRA Git | Maintained and supported by SUSTech CRA and CCSE