Merge tag 'for-linus-hmm' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Address space mirroring's main objective is to allow duplication of a range of

CPU page table into a device page table; HMM helps keep both synchronized. A

device driver that wants to mirror a process address space must start with the

registration of an hmm_mirror struct::

registration of a mmu_interval_notifier::

 int hmm_mirror_register(struct hmm_mirror *mirror,

 mni->ops = &driver_ops;

 int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni,

			          unsigned long start, unsigned long length,

			          struct mm_struct *mm);

The mirror struct has a set of callbacks that are used

to propagate CPU page tables::

 struct hmm_mirror_ops {

     /* release() - release hmm_mirror

      *

      * @mirror: pointer to struct hmm_mirror

      *

      * This is called when the mm_struct is being released.  The callback

      * must ensure that all access to any pages obtained from this mirror

      * is halted before the callback returns. All future access should

      * fault.

      */

     void (*release)(struct hmm_mirror *mirror);

     /* sync_cpu_device_pagetables() - synchronize page tables

      *

      * @mirror: pointer to struct hmm_mirror

      * @update: update information (see struct mmu_notifier_range)

      * Return: -EAGAIN if update.blockable false and callback need to

      *         block, 0 otherwise.

      *

      * This callback ultimately originates from mmu_notifiers when the CPU

      * page table is updated. The device driver must update its page table

      * in response to this callback. The update argument tells what action

      * to perform.

      *

      * The device driver must not return from this callback until the device

      * page tables are completely updated (TLBs flushed, etc); this is a

      * synchronous call.

      */

     int (*sync_cpu_device_pagetables)(struct hmm_mirror *mirror,

                                       const struct hmm_update *update);

 };

The device driver must perform the update action to the range (mark range

read only, or fully unmap, etc.). The device must complete the update before

the driver callback returns.

During the driver_ops->invalidate() callback the device driver must perform

the update action to the range (mark range read only, or fully unmap,

etc.). The device must complete the update before the driver callback returns.

When the device driver wants to populate a range of virtual addresses, it can

use::

      struct hmm_range range;

      ...

      range.notifier = &mni;

      range.start = ...;

      range.end = ...;

      range.pfns = ...;

      range.flags = ...;

      range.values = ...;

      range.pfn_shift = ...;

      hmm_range_register(&range, mirror);

      /*

       * Just wait for range to be valid, safe to ignore return value as we

       * will use the return value of hmm_range_fault() below under the

       * mmap_sem to ascertain the validity of the range.

       */

      hmm_range_wait_until_valid(&range, TIMEOUT_IN_MSEC);

      if (!mmget_not_zero(mni->notifier.mm))

          return -EFAULT;

 again:

      range.notifier_seq = mmu_interval_read_begin(&mni);

      down_read(&mm->mmap_sem);

      ret = hmm_range_fault(&range, HMM_RANGE_SNAPSHOT);

      if (ret) {

          up_read(&mm->mmap_sem);

          if (ret == -EBUSY) {

            /*

             * No need to check hmm_range_wait_until_valid() return value

             * on retry we will get proper error with hmm_range_fault()

             */

            hmm_range_wait_until_valid(&range, TIMEOUT_IN_MSEC);

          if (ret == -EBUSY)

                 goto again;

          }

          hmm_range_unregister(&range);

          return ret;

      }

      up_read(&mm->mmap_sem);

      take_lock(driver->update);

      if (!hmm_range_valid(&range)) {

      if (mmu_interval_read_retry(&ni, range.notifier_seq) {

          release_lock(driver->update);

          up_read(&mm->mmap_sem);

          goto again;

      }

      // Use pfns array content to update device page table

      /* Use pfns array content to update device page table,

       * under the update lock */

      hmm_range_unregister(&range);

      release_lock(driver->update);

      up_read(&mm->mmap_sem);

      return 0;

 }

The driver->update lock is the same lock that the driver takes inside its

sync_cpu_device_pagetables() callback. That lock must be held before calling

hmm_range_valid() to avoid any race with a concurrent CPU page table update.

HMM implements all this on top of the mmu_notifier API because we wanted a

simpler API and also to be able to perform optimizations latter on like doing

concurrent device updates in multi-devices scenario.

HMM also serves as an impedance mismatch between how CPU page table updates

are done (by CPU write to the page table and TLB flushes) and how devices

update their own page table. Device updates are a multi-step process. First,

appropriate commands are written to a buffer, then this buffer is scheduled for

execution on the device. It is only once the device has executed commands in

the buffer that the update is done. Creating and scheduling the update command

buffer can happen concurrently for multiple devices. Waiting for each device to

report commands as executed is serialized (there is no point in doing this

concurrently).

invalidate() callback. That lock must be held before calling

mmu_interval_read_retry() to avoid any race with a concurrent CPU page table

update.

Leverage default_flags and pfn_flags_mask

=========================================

	struct mutex  lock_reset;

	struct amdgpu_doorbell_index doorbell_index;

	struct mutex			notifier_lock;

	int asic_reset_res;

	struct work_struct		xgmi_reset_work;

 *

 * Returns 0 for success, negative errno for errors.

 */

static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,

			   uint64_t user_addr)

static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr)

{

	struct amdkfd_process_info *process_info = mem->process_info;

	struct amdgpu_bo *bo = mem->bo;

	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);

	if (user_addr) {

		ret = init_user_pages(*mem, current->mm, user_addr);

		ret = init_user_pages(*mem, user_addr);

		if (ret)

			goto allocate_init_user_pages_failed;

	}

			return ret;

		}

		/*

		 * FIXME: Cannot ignore the return code, must hold

		 * notifier_lock

		 */

		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);

		/* Mark the BO as valid unless it was invalidated

		e->tv.num_shared = 2;

	amdgpu_bo_list_get_list(p->bo_list, &p->validated);

	if (p->bo_list->first_userptr != p->bo_list->num_entries)

		p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX);

	INIT_LIST_HEAD(&duplicates);

	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);

	if (r)

		goto error_unlock;

	/* No memory allocation is allowed while holding the mn lock.

	 * p->mn is hold until amdgpu_cs_submit is finished and fence is added

	 * to BOs.

	/* No memory allocation is allowed while holding the notifier lock.

	 * The lock is held until amdgpu_cs_submit is finished and fence is

	 * added to BOs.

	 */

	amdgpu_mn_lock(p->mn);

	mutex_lock(&p->adev->notifier_lock);

	/* If userptr are invalidated after amdgpu_cs_parser_bos(), return

	 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.

	amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);

	ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);

	amdgpu_mn_unlock(p->mn);

	mutex_unlock(&p->adev->notifier_lock);

	return 0;

error_abort:

	drm_sched_job_cleanup(&job->base);

	amdgpu_mn_unlock(p->mn);

	mutex_unlock(&p->adev->notifier_lock);

error_unlock:

	amdgpu_job_free(job);

	mutex_init(&adev->virt.vf_errors.lock);

	hash_init(adev->mn_hash);

	mutex_init(&adev->lock_reset);

	mutex_init(&adev->notifier_lock);

	mutex_init(&adev->virt.dpm_mutex);

	mutex_init(&adev->psp.mutex);