Commit d174bd64 authored by Daniel Vetter's avatar Daniel Vetter
Browse files

drm/i915: extract copy helpers from shmem_pread|pwrite 



While moving around things, this two functions slowly grew out of any
sane bounds. So extract a few lines that do the copying and
clflushing. Also add a few comments to explain what's going on.

v2: Again do s/needs_clflush/needs_clflush_after/ in the write paths
as suggested by Chris Wilson.

Tested-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Signed-Off-by: default avatarDaniel Vetter <daniel.vetter@ffwll.ch>
parent 117babcd

drivers/gpu/drm/i915/i915_gem.c

+136 −60
Original line number Diff line number Diff line
@@ -287,6 +287,60 @@ __copy_from_user_swizzled(char __user *gpu_vaddr, int gpu_offset, 
	return 0;

}



/* Per-page copy function for the shmem pread fastpath.

 * Flushes invalid cachelines before reading the target if

 * needs_clflush is set. */

static int

shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,

		 char __user *user_data,

		 bool page_do_bit17_swizzling, bool needs_clflush)

{

	char *vaddr;

	int ret;



	if (page_do_bit17_swizzling)

		return -EINVAL;



	vaddr = kmap_atomic(page);

	if (needs_clflush)

		drm_clflush_virt_range(vaddr + shmem_page_offset,

				       page_length);

	ret = __copy_to_user_inatomic(user_data,

				      vaddr + shmem_page_offset,

				      page_length);

	kunmap_atomic(vaddr);



	return ret;

}



/* Only difference to the fast-path function is that this can handle bit17

 * and uses non-atomic copy and kmap functions. */

static int

shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,

		 char __user *user_data,

		 bool page_do_bit17_swizzling, bool needs_clflush)

{

	char *vaddr;

	int ret;



	vaddr = kmap(page);

	if (needs_clflush)

		drm_clflush_virt_range(vaddr + shmem_page_offset,

				       page_length);



	if (page_do_bit17_swizzling)

		ret = __copy_to_user_swizzled(user_data,

					      vaddr, shmem_page_offset,

					      page_length);

	else

		ret = __copy_to_user(user_data,

				     vaddr + shmem_page_offset,

				     page_length);

	kunmap(page);



	return ret;

}



static int

i915_gem_shmem_pread(struct drm_device *dev,

		     struct drm_i915_gem_object *obj,
@@ -325,7 +379,6 @@ i915_gem_shmem_pread(struct drm_device *dev, 


	while (remain > 0) {

		struct page *page;

		char *vaddr;



		/* Operation in this page

		 *
@@ -352,18 +405,11 @@ i915_gem_shmem_pread(struct drm_device *dev, 
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&

			(page_to_phys(page) & (1 << 17)) != 0;



		if (!page_do_bit17_swizzling) {

			vaddr = kmap_atomic(page);

			if (needs_clflush)

				drm_clflush_virt_range(vaddr + shmem_page_offset,

						       page_length);

			ret = __copy_to_user_inatomic(user_data,

						      vaddr + shmem_page_offset,

						      page_length);

			kunmap_atomic(vaddr);

		ret = shmem_pread_fast(page, shmem_page_offset, page_length,

				       user_data, page_do_bit17_swizzling,

				       needs_clflush);

		if (ret == 0)

			goto next_page;

		}



		hit_slowpath = 1;

		page_cache_get(page);
@@ -379,20 +425,9 @@ i915_gem_shmem_pread(struct drm_device *dev, 
			prefaulted = 1;

		}



		vaddr = kmap(page);

		if (needs_clflush)

			drm_clflush_virt_range(vaddr + shmem_page_offset,

					       page_length);



		if (page_do_bit17_swizzling)

			ret = __copy_to_user_swizzled(user_data,

						      vaddr, shmem_page_offset,

						      page_length);

		else

			ret = __copy_to_user(user_data,

					     vaddr + shmem_page_offset,

					     page_length);

		kunmap(page);

		ret = shmem_pread_slow(page, shmem_page_offset, page_length,

				       user_data, page_do_bit17_swizzling,

				       needs_clflush);



		mutex_lock(&dev->struct_mutex);

		page_cache_release(page);
@@ -557,6 +592,70 @@ out: 
	return ret;

}



/* Per-page copy function for the shmem pwrite fastpath.

 * Flushes invalid cachelines before writing to the target if

 * needs_clflush_before is set and flushes out any written cachelines after

 * writing if needs_clflush is set. */

static int

shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,

		  char __user *user_data,

		  bool page_do_bit17_swizzling,

		  bool needs_clflush_before,

		  bool needs_clflush_after)

{

	char *vaddr;

	int ret;



	if (page_do_bit17_swizzling)

		return -EINVAL;



	vaddr = kmap_atomic(page);

	if (needs_clflush_before)

		drm_clflush_virt_range(vaddr + shmem_page_offset,

				       page_length);

	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,

						user_data,

						page_length);

	if (needs_clflush_after)

		drm_clflush_virt_range(vaddr + shmem_page_offset,

				       page_length);

	kunmap_atomic(vaddr);



	return ret;

}



/* Only difference to the fast-path function is that this can handle bit17

 * and uses non-atomic copy and kmap functions. */

static int

shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,

		  char __user *user_data,

		  bool page_do_bit17_swizzling,

		  bool needs_clflush_before,

		  bool needs_clflush_after)

{

	char *vaddr;

	int ret;



	vaddr = kmap(page);

	if (needs_clflush_before)

		drm_clflush_virt_range(vaddr + shmem_page_offset,

				       page_length);

	if (page_do_bit17_swizzling)

		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,

						user_data,

						page_length);

	else

		ret = __copy_from_user(vaddr + shmem_page_offset,

				       user_data,

				       page_length);

	if (needs_clflush_after)

		drm_clflush_virt_range(vaddr + shmem_page_offset,

				       page_length);

	kunmap(page);



	return ret;

}



static int

i915_gem_shmem_pwrite(struct drm_device *dev,

		      struct drm_i915_gem_object *obj,
@@ -601,7 +700,6 @@ i915_gem_shmem_pwrite(struct drm_device *dev, 


	while (remain > 0) {

		struct page *page;

		char *vaddr;

		int partial_cacheline_write;



		/* Operation in this page
@@ -637,43 +735,21 @@ i915_gem_shmem_pwrite(struct drm_device *dev, 
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&

			(page_to_phys(page) & (1 << 17)) != 0;



		if (!page_do_bit17_swizzling) {

			vaddr = kmap_atomic(page);

			if (partial_cacheline_write)

				drm_clflush_virt_range(vaddr + shmem_page_offset,

						       page_length);

			ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,

								user_data,

								page_length);

			if (needs_clflush_after)

				drm_clflush_virt_range(vaddr + shmem_page_offset,

						       page_length);

			kunmap_atomic(vaddr);



		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,

					user_data, page_do_bit17_swizzling,

					partial_cacheline_write,

					needs_clflush_after);

		if (ret == 0)

			goto next_page;

		}



		hit_slowpath = 1;

		page_cache_get(page);

		mutex_unlock(&dev->struct_mutex);



		vaddr = kmap(page);

		if (partial_cacheline_write)

			drm_clflush_virt_range(vaddr + shmem_page_offset,

					       page_length);

		if (page_do_bit17_swizzling)

			ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,

							user_data,

							page_length);

		else

			ret = __copy_from_user(vaddr + shmem_page_offset,

					       user_data,

					       page_length);

		if (needs_clflush_after)

			drm_clflush_virt_range(vaddr + shmem_page_offset,

					       page_length);

		kunmap(page);

		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,

					user_data, page_do_bit17_swizzling,

					partial_cacheline_write,

					needs_clflush_after);



		mutex_lock(&dev->struct_mutex);

		page_cache_release(page);

CRA Git | Maintained and supported by SUSTech CRA and CCSE