IB/mlx5: Page faults handling infrastructure (6aec21f6) · Commits · 戴 / test

drivers/infiniband/hw/mlx5/main.c

+27 −4

Original line number	Diff line number	Diff line
		@@ -864,7 +864,7 @@ static ssize_t show_reg_pages(struct device *device,
		struct mlx5_ib_dev *dev =
		container_of(device, struct mlx5_ib_dev, ib_dev.dev);

		return sprintf(buf, "%d\n", dev->mdev->priv.reg_pages);
		return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
		}

		static ssize_t show_hca(struct device device, struct device_attribute attr,
		@@ -1389,16 +1389,19 @@ static void mlx5_ib_add(struct mlx5_core_dev mdev)
		goto err_eqs;

		mutex_init(&dev->cap_mask_mutex);
		spin_lock_init(&dev->mr_lock);

		err = create_dev_resources(&dev->devr);
		if (err)
		goto err_eqs;

		err = ib_register_device(&dev->ib_dev, NULL);
		err = mlx5_ib_odp_init_one(dev);
		if (err)
		goto err_rsrc;

		err = ib_register_device(&dev->ib_dev, NULL);
		if (err)
		goto err_odp;

		err = create_umr_res(dev);
		if (err)
		goto err_dev;
		@@ -1420,6 +1423,9 @@ err_umrc:
		err_dev:
		ib_unregister_device(&dev->ib_dev);

		err_odp:
		mlx5_ib_odp_remove_one(dev);

		err_rsrc:
		destroy_dev_resources(&dev->devr);

		@@ -1435,8 +1441,10 @@ err_dealloc:
		static void mlx5_ib_remove(struct mlx5_core_dev mdev, void context)
		{
		struct mlx5_ib_dev *dev = context;

		ib_unregister_device(&dev->ib_dev);
		destroy_umrc_res(dev);
		mlx5_ib_odp_remove_one(dev);
		destroy_dev_resources(&dev->devr);
		free_comp_eqs(dev);
		ib_dealloc_device(&dev->ib_dev);
		@@ -1450,15 +1458,30 @@ static struct mlx5_interface mlx5_ib_interface = {

		static int __init mlx5_ib_init(void)
		{
		int err;

		if (deprecated_prof_sel != 2)
		pr_warn("prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n");

		return mlx5_register_interface(&mlx5_ib_interface);
		err = mlx5_ib_odp_init();
		if (err)
		return err;

		err = mlx5_register_interface(&mlx5_ib_interface);
		if (err)
		goto clean_odp;

		return err;

		clean_odp:
		mlx5_ib_odp_cleanup();
		return err;
		}

		static void __exit mlx5_ib_cleanup(void)
		{
		mlx5_unregister_interface(&mlx5_ib_interface);
		mlx5_ib_odp_cleanup();
		}

		module_init(mlx5_ib_init);

drivers/infiniband/hw/mlx5/mlx5_ib.h

+65 −2

Original line number	Diff line number	Diff line
		@@ -149,6 +149,29 @@ enum {
		MLX5_QP_EMPTY
		};

		/*
		* Connect-IB can trigger up to four concurrent pagefaults
		* per-QP.
		*/
		enum mlx5_ib_pagefault_context {
		MLX5_IB_PAGEFAULT_RESPONDER_READ,
		MLX5_IB_PAGEFAULT_REQUESTOR_READ,
		MLX5_IB_PAGEFAULT_RESPONDER_WRITE,
		MLX5_IB_PAGEFAULT_REQUESTOR_WRITE,
		MLX5_IB_PAGEFAULT_CONTEXTS
		};

		static inline enum mlx5_ib_pagefault_context
		mlx5_ib_get_pagefault_context(struct mlx5_pagefault *pagefault)
		{
		return pagefault->flags & (MLX5_PFAULT_REQUESTOR \| MLX5_PFAULT_WRITE);
		}

		struct mlx5_ib_pfault {
		struct work_struct work;
		struct mlx5_pagefault mpfault;
		};

		struct mlx5_ib_qp {
		struct ib_qp ibqp;
		struct mlx5_core_qp mqp;
		@@ -194,6 +217,21 @@ struct mlx5_ib_qp {

		/* Store signature errors */
		bool signature_en;

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		/*
		* A flag that is true for QP's that are in a state that doesn't
		* allow page faults, and shouldn't schedule any more faults.
		*/
		int disable_page_faults;
		/*
		* The disable_page_faults_lock protects a QP's disable_page_faults
		* field, allowing for a thread to atomically check whether the QP
		* allows page faults, and if so schedule a page fault.
		*/
		spinlock_t disable_page_faults_lock;
		struct mlx5_ib_pfault pagefaults[MLX5_IB_PAGEFAULT_CONTEXTS];
		#endif
		};

		struct mlx5_ib_cq_buf {
		@@ -392,13 +430,17 @@ struct mlx5_ib_dev {
		struct umr_common umrc;
		/* sync used page count stats
		*/
		spinlock_t mr_lock;
		struct mlx5_ib_resources devr;
		struct mlx5_mr_cache cache;
		struct timer_list delay_timer;
		int fill_delay;
		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		struct ib_odp_caps odp_caps;
		/*
		* Sleepable RCU that prevents destruction of MRs while they are still
		* being used by a page fault handler.
		*/
		struct srcu_struct mr_srcu;
		#endif
		};

		@@ -575,12 +617,33 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
		struct ib_mr_status *mr_status);

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		extern struct workqueue_struct *mlx5_ib_page_fault_wq;

		int mlx5_ib_internal_query_odp_caps(struct mlx5_ib_dev *dev);
		#else
		void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault);
		void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp);
		int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
		void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev);
		int __init mlx5_ib_odp_init(void);
		void mlx5_ib_odp_cleanup(void);
		void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp);
		void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp);

		#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
		static inline int mlx5_ib_internal_query_odp_caps(struct mlx5_ib_dev *dev)
		{
		return 0;
		}

		static inline void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp) {}
		static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
		static inline void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev) {}
		static inline int mlx5_ib_odp_init(void) { return 0; }
		static inline void mlx5_ib_odp_cleanup(void) {}
		static inline void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp) {}
		static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp) {}

		#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */

		static inline void init_query_mad(struct ib_smp *mad)

drivers/infiniband/hw/mlx5/mr.c

+31 −14

Original line number	Diff line number	Diff line
		@@ -52,6 +52,8 @@ static __be64 mlx5_ib_update_mtt_emergency_buffer[
		static DEFINE_MUTEX(mlx5_ib_update_mtt_emergency_buffer_mutex);
		#endif

		static int clean_mr(struct mlx5_ib_mr *mr);

		static int order2idx(struct mlx5_ib_dev *dev, int order)
		{
		struct mlx5_mr_cache *cache = &dev->cache;
		@@ -1049,6 +1051,10 @@ struct ib_mr mlx5_ib_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
		mlx5_ib_dbg(dev, "cache empty for order %d", order);
		mr = NULL;
		}
		} else if (access_flags & IB_ACCESS_ON_DEMAND) {
		err = -EINVAL;
		pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB");
		goto error;
		}

		if (!mr)
		@@ -1064,9 +1070,7 @@ struct ib_mr mlx5_ib_reg_user_mr(struct ib_pd pd, u64 start, u64 length,

		mr->umem = umem;
		mr->npages = npages;
		spin_lock(&dev->mr_lock);
		dev->mdev->priv.reg_pages += npages;
		spin_unlock(&dev->mr_lock);
		atomic_add(npages, &dev->mdev->priv.reg_pages);
		mr->ibmr.lkey = mr->mmr.key;
		mr->ibmr.rkey = mr->mmr.key;

		@@ -1110,12 +1114,9 @@ error:
		return err;
		}

		int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
		static int clean_mr(struct mlx5_ib_mr *mr)
		{
		struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
		struct mlx5_ib_mr *mr = to_mmr(ibmr);
		struct ib_umem *umem = mr->umem;
		int npages = mr->npages;
		struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
		int umred = mr->umred;
		int err;

		@@ -1135,16 +1136,32 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
		free_cached_mr(dev, mr);
		}

		if (!umred)
		kfree(mr);

		return 0;
		}

		int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
		{
		struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
		struct mlx5_ib_mr *mr = to_mmr(ibmr);
		int npages = mr->npages;
		struct ib_umem *umem = mr->umem;

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		if (umem)
		/* Wait for all running page-fault handlers to finish. */
		synchronize_srcu(&dev->mr_srcu);
		#endif

		clean_mr(mr);

		if (umem) {
		ib_umem_release(umem);
		spin_lock(&dev->mr_lock);
		dev->mdev->priv.reg_pages -= npages;
		spin_unlock(&dev->mr_lock);
		atomic_sub(npages, &dev->mdev->priv.reg_pages);
		}

		if (!umred)
		kfree(mr);

		return 0;
		}

drivers/infiniband/hw/mlx5/odp.c

+145 −0

Original line number	Diff line number	Diff line
		@@ -32,6 +32,8 @@

		#include "mlx5_ib.h"

		struct workqueue_struct *mlx5_ib_page_fault_wq;

		#define COPY_ODP_BIT_MLX_TO_IB(reg, ib_caps, field_name, bit_name) do { \
		if (be32_to_cpu(reg.field_name) & MLX5_ODP_SUPPORT_##bit_name) \
		ib_caps->field_name \|= IB_ODP_SUPPORT_##bit_name; \
		@@ -58,3 +60,146 @@ int mlx5_ib_internal_query_odp_caps(struct mlx5_ib_dev *dev)
		out:
		return err;
		}

		static struct mlx5_ib_mr mlx5_ib_odp_find_mr_lkey(struct mlx5_ib_dev dev,
		u32 key)
		{
		u32 base_key = mlx5_base_mkey(key);
		struct mlx5_core_mr *mmr = __mlx5_mr_lookup(dev->mdev, base_key);

		if (!mmr \|\| mmr->key != key)
		return NULL;

		return container_of(mmr, struct mlx5_ib_mr, mmr);
		}

		static void mlx5_ib_page_fault_resume(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault,
		int error) {
		struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);
		int ret = mlx5_core_page_fault_resume(dev->mdev, qp->mqp.qpn,
		pfault->mpfault.flags,
		error);
		if (ret)
		pr_err("Failed to resolve the page fault on QP 0x%x\n",
		qp->mqp.qpn);
		}

		void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,
		struct mlx5_ib_pfault *pfault)
		{
		u8 event_subtype = pfault->mpfault.event_subtype;

		switch (event_subtype) {
		default:
		pr_warn("Invalid page fault event subtype: 0x%x\n",
		event_subtype);
		mlx5_ib_page_fault_resume(qp, pfault, 1);
		break;
		}
		}

		static void mlx5_ib_qp_pfault_action(struct work_struct *work)
		{
		struct mlx5_ib_pfault *pfault = container_of(work,
		struct mlx5_ib_pfault,
		work);
		enum mlx5_ib_pagefault_context context =
		mlx5_ib_get_pagefault_context(&pfault->mpfault);
		struct mlx5_ib_qp *qp = container_of(pfault, struct mlx5_ib_qp,
		pagefaults[context]);
		mlx5_ib_mr_pfault_handler(qp, pfault);
		}

		void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp)
		{
		unsigned long flags;

		spin_lock_irqsave(&qp->disable_page_faults_lock, flags);
		qp->disable_page_faults = 1;
		spin_unlock_irqrestore(&qp->disable_page_faults_lock, flags);

		/*
		* Note that at this point, we are guarenteed that no more
		* work queue elements will be posted to the work queue with
		* the QP we are closing.
		*/
		flush_workqueue(mlx5_ib_page_fault_wq);
		}

		void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp)
		{
		unsigned long flags;

		spin_lock_irqsave(&qp->disable_page_faults_lock, flags);
		qp->disable_page_faults = 0;
		spin_unlock_irqrestore(&qp->disable_page_faults_lock, flags);
		}

		static void mlx5_ib_pfault_handler(struct mlx5_core_qp *qp,
		struct mlx5_pagefault *pfault)
		{
		/*
		* Note that we will only get one fault event per QP per context
		* (responder/initiator, read/write), until we resolve the page fault
		* with the mlx5_ib_page_fault_resume command. Since this function is
		* called from within the work element, there is no risk of missing
		* events.
		*/
		struct mlx5_ib_qp *mibqp = to_mibqp(qp);
		enum mlx5_ib_pagefault_context context =
		mlx5_ib_get_pagefault_context(pfault);
		struct mlx5_ib_pfault *qp_pfault = &mibqp->pagefaults[context];

		qp_pfault->mpfault = *pfault;

		/* No need to stop interrupts here since we are in an interrupt */
		spin_lock(&mibqp->disable_page_faults_lock);
		if (!mibqp->disable_page_faults)
		queue_work(mlx5_ib_page_fault_wq, &qp_pfault->work);
		spin_unlock(&mibqp->disable_page_faults_lock);
		}

		void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp)
		{
		int i;

		qp->disable_page_faults = 1;
		spin_lock_init(&qp->disable_page_faults_lock);

		qp->mqp.pfault_handler = mlx5_ib_pfault_handler;

		for (i = 0; i < MLX5_IB_PAGEFAULT_CONTEXTS; ++i)
		INIT_WORK(&qp->pagefaults[i].work, mlx5_ib_qp_pfault_action);
		}

		int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev)
		{
		int ret;

		ret = init_srcu_struct(&ibdev->mr_srcu);
		if (ret)
		return ret;

		return 0;
		}

		void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev)
		{
		cleanup_srcu_struct(&ibdev->mr_srcu);
		}

		int __init mlx5_ib_odp_init(void)
		{
		mlx5_ib_page_fault_wq =
		create_singlethread_workqueue("mlx5_ib_page_faults");
		if (!mlx5_ib_page_fault_wq)
		return -ENOMEM;

		return 0;
		}

		void mlx5_ib_odp_cleanup(void)
		{
		destroy_workqueue(mlx5_ib_page_fault_wq);
		}

drivers/infiniband/hw/mlx5/qp.c

+25 −1

Original line number	Diff line number	Diff line
		@@ -876,6 +876,8 @@ static int create_qp_common(struct mlx5_ib_dev dev, struct ib_pd pd,
		int inlen = sizeof(*in);
		int err;

		mlx5_ib_odp_create_qp(qp);

		gen = &dev->mdev->caps.gen;
		mutex_init(&qp->mutex);
		spin_lock_init(&qp->sq.lock);
		@@ -1160,11 +1162,13 @@ static void destroy_qp_common(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp)
		in = kzalloc(sizeof(*in), GFP_KERNEL);
		if (!in)
		return;
		if (qp->state != IB_QPS_RESET)
		if (qp->state != IB_QPS_RESET) {
		mlx5_ib_qp_disable_pagefaults(qp);
		if (mlx5_core_qp_modify(dev->mdev, to_mlx5_state(qp->state),
		MLX5_QP_STATE_RST, in, sizeof(*in), &qp->mqp))
		mlx5_ib_warn(dev, "mlx5_ib: modify QP %06x to RESET failed\n",
		qp->mqp.qpn);
		}

		get_cqs(qp, &send_cq, &recv_cq);

		@@ -1712,6 +1716,15 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
		if (mlx5_st < 0)
		goto out;

		/* If moving to a reset or error state, we must disable page faults on
		* this QP and flush all current page faults. Otherwise a stale page
		* fault may attempt to work on this QP after it is reset and moved
		* again to RTS, and may cause the driver and the device to get out of
		* sync. */
		if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
		(new_state == IB_QPS_RESET \|\| new_state == IB_QPS_ERR))
		mlx5_ib_qp_disable_pagefaults(qp);

		optpar = ib_mask_to_mlx5_opt(attr_mask);
		optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
		in->optparam = cpu_to_be32(optpar);
		@@ -1721,6 +1734,9 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
		if (err)
		goto out;

		if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
		mlx5_ib_qp_enable_pagefaults(qp);

		qp->state = new_state;

		if (attr_mask & IB_QP_ACCESS_FLAGS)
		@@ -3026,6 +3042,14 @@ int mlx5_ib_query_qp(struct ib_qp ibqp, struct ib_qp_attr qp_attr, int qp_attr
		int mlx5_state;
		int err = 0;

		#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		/*
		* Wait for any outstanding page faults, in case the user frees memory
		* based upon this query's result.
		*/
		flush_workqueue(mlx5_ib_page_fault_wq);
		#endif

		mutex_lock(&qp->mutex);
		outb = kzalloc(sizeof(*outb), GFP_KERNEL);
		if (!outb) {

Admin message