Commit c7d8b782 authored by Jason Gunthorpe's avatar Jason Gunthorpe
Browse files

hmm: use mmu_notifier_get/put for 'struct hmm' 

This is a significant simplification, it eliminates all the remaining
'hmm' stuff in mm_struct, eliminates krefing along the critical notifier
paths, and takes away all the ugly locking and abuse of page_table_lock.

mmu_notifier_get() provides the single struct hmm per struct mm which
eliminates mm->hmm.

It also directly guarantees that no mmu_notifier op callback is callable
while concurrent free is possible, this eliminates all the krefs inside
the mmu_notifier callbacks.

The remaining krefs in the range code were overly cautious, drivers are
already not permitted to free the mirror while a range exists.

Link: https://lore.kernel.org/r/20190806231548.25242-6-jgg@ziepe.ca


Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarRalph Campbell <rcampbell@nvidia.com>
Tested-by: default avatarRalph Campbell <rcampbell@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent e4c057d0

drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c

+2 −0
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ 
#include <linux/pm_runtime.h>

#include <linux/vga_switcheroo.h>

#include <drm/drm_probe_helper.h>

#include <linux/mmu_notifier.h>



#include "amdgpu.h"

#include "amdgpu_irq.h"
@@ -1464,6 +1465,7 @@ static void __exit amdgpu_exit(void) 
	amdgpu_unregister_atpx_handler();

	amdgpu_sync_fini();

	amdgpu_fence_slab_fini();

	mmu_notifier_synchronize();

}



module_init(amdgpu_init);

drivers/gpu/drm/nouveau/nouveau_drm.c

+3 −0
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ 
#include <linux/pci.h>

#include <linux/pm_runtime.h>

#include <linux/vga_switcheroo.h>

#include <linux/mmu_notifier.h>



#include <drm/drmP.h>

#include <drm/drm_crtc_helper.h>
@@ -1292,6 +1293,8 @@ nouveau_drm_exit(void) 
#ifdef CONFIG_NOUVEAU_PLATFORM_DRIVER

	platform_driver_unregister(&nouveau_platform_driver);

#endif

	if (IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM))

		mmu_notifier_synchronize();

}



module_init(nouveau_drm_init);

include/linux/hmm.h

+1 −11
Original line number Diff line number Diff line
@@ -84,15 +84,12 @@ 
 * @notifiers: count of active mmu notifiers

 */

struct hmm {

	struct mm_struct	*mm;

	struct kref		kref;

	struct mmu_notifier	mmu_notifier;

	spinlock_t		ranges_lock;

	struct list_head	ranges;

	struct list_head	mirrors;

	struct mmu_notifier	mmu_notifier;

	struct rw_semaphore	mirrors_sem;

	wait_queue_head_t	wq;

	struct rcu_head		rcu;

	long			notifiers;

};
@@ -409,13 +406,6 @@ long hmm_range_dma_unmap(struct hmm_range *range, 
 */

#define HMM_RANGE_DEFAULT_TIMEOUT 1000



/* Below are for HMM internal use only! Not to be used by device driver! */

static inline void hmm_mm_init(struct mm_struct *mm)

{

	mm->hmm = NULL;

}

#else /* IS_ENABLED(CONFIG_HMM_MIRROR) */

static inline void hmm_mm_init(struct mm_struct *mm) {}

#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */



#endif /* LINUX_HMM_H */

include/linux/mm_types.h

+0 −6
Original line number Diff line number Diff line
@@ -25,7 +25,6 @@ 


struct address_space;

struct mem_cgroup;

struct hmm;



/*

 * Each physical page in the system has a struct page associated with
@@ -502,11 +501,6 @@ struct mm_struct { 
		atomic_long_t hugetlb_usage;

#endif

		struct work_struct async_put_work;



#ifdef CONFIG_HMM_MIRROR

		/* HMM needs to track a few things per mm */

		struct hmm *hmm;

#endif

	} __randomize_layout;



	/*

kernel/fork.c

+0 −1
Original line number Diff line number Diff line
@@ -1007,7 +1007,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, 
	mm_init_owner(mm, p);

	RCU_INIT_POINTER(mm->exe_file, NULL);

	mmu_notifier_mm_init(mm);

	hmm_mm_init(mm);

	init_tlb_flush_pending(mm);

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS

	mm->pmd_huge_pte = NULL;

CRA Git | Maintained and supported by SUSTech CRA and CCSE