RDS/IW: Convert to new memory registration API (9ddc8737) · Commits · 戴 / test

net/rds/iw.h

+2 −3

Original line number	Diff line number	Diff line
		@@ -74,13 +74,12 @@ struct rds_iw_send_work {
		struct rm_rdma_op *s_op;
		struct rds_iw_mapping *s_mapping;
		struct ib_mr *s_mr;
		struct ib_fast_reg_page_list *s_page_list;
		unsigned char s_remap_count;

		union {
		struct ib_send_wr s_send_wr;
		struct ib_rdma_wr s_rdma_wr;
		struct ib_fast_reg_wr s_fast_reg_wr;
		struct ib_reg_wr s_reg_wr;
		};
		struct ib_sge s_sge[RDS_IW_MAX_SGE];
		unsigned long s_queued;
		@@ -199,7 +198,7 @@ struct rds_iw_device {

		/* Magic WR_ID for ACKs */
		#define RDS_IW_ACK_WR_ID ((u64)0xffffffffffffffffULL)
		#define RDS_IW_FAST_REG_WR_ID ((u64)0xefefefefefefefefULL)
		#define RDS_IW_REG_WR_ID ((u64)0xefefefefefefefefULL)
		#define RDS_IW_LOCAL_INV_WR_ID ((u64)0xdfdfdfdfdfdfdfdfULL)

		struct rds_iw_statistics {

net/rds/iw_rdma.c

+45 −83

Original line number	Diff line number	Diff line
		@@ -47,7 +47,6 @@ struct rds_iw_mr {
		struct rdma_cm_id *cm_id;

		struct ib_mr *mr;
		struct ib_fast_reg_page_list *page_list;

		struct rds_iw_mapping mapping;
		unsigned char remap_count;
		@@ -77,8 +76,8 @@ struct rds_iw_mr_pool {

		static int rds_iw_flush_mr_pool(struct rds_iw_mr_pool *pool, int free_all);
		static void rds_iw_mr_pool_flush_worker(struct work_struct *work);
		static int rds_iw_init_fastreg(struct rds_iw_mr_pool pool, struct rds_iw_mr ibmr);
		static int rds_iw_map_fastreg(struct rds_iw_mr_pool *pool,
		static int rds_iw_init_reg(struct rds_iw_mr_pool pool, struct rds_iw_mr ibmr);
		static int rds_iw_map_reg(struct rds_iw_mr_pool *pool,
		struct rds_iw_mr *ibmr,
		struct scatterlist *sg, unsigned int nents);
		static void rds_iw_free_fastreg(struct rds_iw_mr_pool pool, struct rds_iw_mr ibmr);
		@@ -258,19 +257,18 @@ static void rds_iw_set_scatterlist(struct rds_iw_scatterlist *sg,
		sg->bytes = 0;
		}

		static u64 rds_iw_map_scatterlist(struct rds_iw_device rds_iwdev,
		static int rds_iw_map_scatterlist(struct rds_iw_device *rds_iwdev,
		struct rds_iw_scatterlist *sg)
		{
		struct ib_device *dev = rds_iwdev->dev;
		u64 *dma_pages = NULL;
		int i, j, ret;
		int i, ret;

		WARN_ON(sg->dma_len);

		sg->dma_len = ib_dma_map_sg(dev, sg->list, sg->len, DMA_BIDIRECTIONAL);
		if (unlikely(!sg->dma_len)) {
		printk(KERN_WARNING "RDS/IW: dma_map_sg failed!\n");
		return ERR_PTR(-EBUSY);
		return -EBUSY;
		}

		sg->bytes = 0;
		@@ -303,31 +301,14 @@ static u64 rds_iw_map_scatterlist(struct rds_iw_device rds_iwdev,
		if (sg->dma_npages > fastreg_message_size)
		goto out_unmap;

		dma_pages = kmalloc(sizeof(u64) * sg->dma_npages, GFP_ATOMIC);
		if (!dma_pages) {
		ret = -ENOMEM;
		goto out_unmap;
		}

		for (i = j = 0; i < sg->dma_len; ++i) {
		unsigned int dma_len = ib_sg_dma_len(dev, &sg->list[i]);
		u64 dma_addr = ib_sg_dma_address(dev, &sg->list[i]);
		u64 end_addr;

		end_addr = dma_addr + dma_len;
		dma_addr &= ~PAGE_MASK;
		for (; dma_addr < end_addr; dma_addr += PAGE_SIZE)
		dma_pages[j++] = dma_addr;
		BUG_ON(j > sg->dma_npages);
		}

		return dma_pages;
		return 0;

		out_unmap:
		ib_dma_unmap_sg(rds_iwdev->dev, sg->list, sg->len, DMA_BIDIRECTIONAL);
		sg->dma_len = 0;
		kfree(dma_pages);
		return ERR_PTR(ret);
		return ret;
		}


		@@ -440,7 +421,7 @@ static struct rds_iw_mr rds_iw_alloc_mr(struct rds_iw_device rds_iwdev)
		INIT_LIST_HEAD(&ibmr->mapping.m_list);
		ibmr->mapping.m_mr = ibmr;

		err = rds_iw_init_fastreg(pool, ibmr);
		err = rds_iw_init_reg(pool, ibmr);
		if (err)
		goto out_no_cigar;

		@@ -622,7 +603,7 @@ void rds_iw_get_mr(struct scatterlist sg, unsigned long nents,
		ibmr->cm_id = cm_id;
		ibmr->device = rds_iwdev;

		ret = rds_iw_map_fastreg(rds_iwdev->mr_pool, ibmr, sg, nents);
		ret = rds_iw_map_reg(rds_iwdev->mr_pool, ibmr, sg, nents);
		if (ret == 0)
		*key_ret = ibmr->mr->rkey;
		else
		@@ -638,7 +619,7 @@ out:
		}

		/*
		* iWARP fastreg handling
		* iWARP reg handling
		*
		* The life cycle of a fastreg registration is a bit different from
		* FMRs.
		@@ -650,7 +631,7 @@ out:
		* This creates a bit of a problem for us, as we do not have the destination
		* IP in GET_MR, so the connection must be setup prior to the GET_MR call for
		* RDMA to be correctly setup. If a fastreg request is present, rds_iw_xmit
		* will try to queue a LOCAL_INV (if needed) and a FAST_REG_MR work request
		* will try to queue a LOCAL_INV (if needed) and a REG_MR work request
		* before queuing the SEND. When completions for these arrive, they are
		* dispatched to the MR has a bit set showing that RDMa can be performed.
		*
		@@ -659,11 +640,10 @@ out:
		* The expectation there is that this invalidation step includes ALL
		* PREVIOUSLY FREED MRs.
		*/
		static int rds_iw_init_fastreg(struct rds_iw_mr_pool *pool,
		static int rds_iw_init_reg(struct rds_iw_mr_pool *pool,
		struct rds_iw_mr *ibmr)
		{
		struct rds_iw_device *rds_iwdev = pool->device;
		struct ib_fast_reg_page_list *page_list = NULL;
		struct ib_mr *mr;
		int err;

		@@ -676,56 +656,44 @@ static int rds_iw_init_fastreg(struct rds_iw_mr_pool *pool,
		return err;
		}

		/* FIXME - this is overkill, but mapping->m_sg.dma_len/mapping->m_sg.dma_npages
		* is not filled in.
		*/
		page_list = ib_alloc_fast_reg_page_list(rds_iwdev->dev, pool->max_message_size);
		if (IS_ERR(page_list)) {
		err = PTR_ERR(page_list);

		printk(KERN_WARNING "RDS/IW: ib_alloc_fast_reg_page_list failed (err=%d)\n", err);
		ib_dereg_mr(mr);
		return err;
		}

		ibmr->page_list = page_list;
		ibmr->mr = mr;
		return 0;
		}

		static int rds_iw_rdma_build_fastreg(struct rds_iw_mapping *mapping)
		static int rds_iw_rdma_reg_mr(struct rds_iw_mapping *mapping)
		{
		struct rds_iw_mr *ibmr = mapping->m_mr;
		struct ib_fast_reg_wr f_wr;
		struct rds_iw_scatterlist *m_sg = &mapping->m_sg;
		struct ib_reg_wr reg_wr;
		struct ib_send_wr *failed_wr;
		int ret;
		int ret, n;

		n = ib_map_mr_sg_zbva(ibmr->mr, m_sg->list, m_sg->len, PAGE_SIZE);
		if (unlikely(n != m_sg->len))
		return n < 0 ? n : -EINVAL;

		reg_wr.wr.next = NULL;
		reg_wr.wr.opcode = IB_WR_REG_MR;
		reg_wr.wr.wr_id = RDS_IW_REG_WR_ID;
		reg_wr.wr.num_sge = 0;
		reg_wr.mr = ibmr->mr;
		reg_wr.key = mapping->m_rkey;
		reg_wr.access = IB_ACCESS_LOCAL_WRITE \|
		IB_ACCESS_REMOTE_READ \|
		IB_ACCESS_REMOTE_WRITE;

		/*
		* Perform a WR for the fast_reg_mr. Each individual page
		* Perform a WR for the reg_mr. Each individual page
		* in the sg list is added to the fast reg page list and placed
		* inside the fast_reg_mr WR. The key used is a rolling 8bit
		* inside the reg_mr WR. The key used is a rolling 8bit
		* counter, which should guarantee uniqueness.
		*/
		ib_update_fast_reg_key(ibmr->mr, ibmr->remap_count++);
		mapping->m_rkey = ibmr->mr->rkey;

		memset(&f_wr, 0, sizeof(f_wr));
		f_wr.wr.wr_id = RDS_IW_FAST_REG_WR_ID;
		f_wr.wr.opcode = IB_WR_FAST_REG_MR;
		f_wr.length = mapping->m_sg.bytes;
		f_wr.rkey = mapping->m_rkey;
		f_wr.page_list = ibmr->page_list;
		f_wr.page_list_len = mapping->m_sg.dma_len;
		f_wr.page_shift = PAGE_SHIFT;
		f_wr.access_flags = IB_ACCESS_LOCAL_WRITE \|
		IB_ACCESS_REMOTE_READ \|
		IB_ACCESS_REMOTE_WRITE;
		f_wr.iova_start = 0;
		f_wr.wr.send_flags = IB_SEND_SIGNALED;

		failed_wr = &f_wr.wr;
		ret = ib_post_send(ibmr->cm_id->qp, &f_wr.wr, &failed_wr);
		BUG_ON(failed_wr != &f_wr.wr);
		failed_wr = &reg_wr.wr;
		ret = ib_post_send(ibmr->cm_id->qp, &reg_wr.wr, &failed_wr);
		BUG_ON(failed_wr != &reg_wr.wr);
		if (ret)
		printk_ratelimited(KERN_WARNING "RDS/IW: %s:%d ib_post_send returned %d\n",
		__func__, __LINE__, ret);
		@@ -757,7 +725,7 @@ out:
		return ret;
		}

		static int rds_iw_map_fastreg(struct rds_iw_mr_pool *pool,
		static int rds_iw_map_reg(struct rds_iw_mr_pool *pool,
		struct rds_iw_mr *ibmr,
		struct scatterlist *sg,
		unsigned int sg_len)
		@@ -765,13 +733,12 @@ static int rds_iw_map_fastreg(struct rds_iw_mr_pool *pool,
		struct rds_iw_device *rds_iwdev = pool->device;
		struct rds_iw_mapping *mapping = &ibmr->mapping;
		u64 *dma_pages;
		int i, ret = 0;
		int ret = 0;

		rds_iw_set_scatterlist(&mapping->m_sg, sg, sg_len);

		dma_pages = rds_iw_map_scatterlist(rds_iwdev, &mapping->m_sg);
		if (IS_ERR(dma_pages)) {
		ret = PTR_ERR(dma_pages);
		ret = rds_iw_map_scatterlist(rds_iwdev, &mapping->m_sg);
		if (ret) {
		dma_pages = NULL;
		goto out;
		}
		@@ -781,10 +748,7 @@ static int rds_iw_map_fastreg(struct rds_iw_mr_pool *pool,
		goto out;
		}

		for (i = 0; i < mapping->m_sg.dma_npages; ++i)
		ibmr->page_list->page_list[i] = dma_pages[i];

		ret = rds_iw_rdma_build_fastreg(mapping);
		ret = rds_iw_rdma_reg_mr(mapping);
		if (ret)
		goto out;

		@@ -870,8 +834,6 @@ static unsigned int rds_iw_unmap_fastreg_list(struct rds_iw_mr_pool *pool,
		static void rds_iw_destroy_fastreg(struct rds_iw_mr_pool *pool,
		struct rds_iw_mr *ibmr)
		{
		if (ibmr->page_list)
		ib_free_fast_reg_page_list(ibmr->page_list);
		if (ibmr->mr)
		ib_dereg_mr(ibmr->mr);
		}

net/rds/iw_send.c

+28 −29

Original line number	Diff line number	Diff line
		@@ -159,13 +159,6 @@ void rds_iw_send_init_ring(struct rds_iw_connection *ic)
		printk(KERN_WARNING "RDS/IW: ib_alloc_mr failed\n");
		break;
		}

		send->s_page_list = ib_alloc_fast_reg_page_list(
		ic->i_cm_id->device, fastreg_message_size);
		if (IS_ERR(send->s_page_list)) {
		printk(KERN_WARNING "RDS/IW: ib_alloc_fast_reg_page_list failed\n");
		break;
		}
		}
		}

		@@ -177,8 +170,6 @@ void rds_iw_send_clear_ring(struct rds_iw_connection *ic)
		for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
		BUG_ON(!send->s_mr);
		ib_dereg_mr(send->s_mr);
		BUG_ON(!send->s_page_list);
		ib_free_fast_reg_page_list(send->s_page_list);
		if (send->s_send_wr.opcode == 0xdead)
		continue;
		if (send->s_rm)
		@@ -227,7 +218,7 @@ void rds_iw_send_cq_comp_handler(struct ib_cq cq, void context)
		continue;
		}

		if (wc.opcode == IB_WC_FAST_REG_MR && wc.wr_id == RDS_IW_FAST_REG_WR_ID) {
		if (wc.opcode == IB_WC_REG_MR && wc.wr_id == RDS_IW_REG_WR_ID) {
		ic->i_fastreg_posted = 1;
		continue;
		}
		@@ -252,7 +243,7 @@ void rds_iw_send_cq_comp_handler(struct ib_cq cq, void context)
		if (send->s_rm)
		rds_iw_send_unmap_rm(ic, send, wc.status);
		break;
		case IB_WR_FAST_REG_MR:
		case IB_WR_REG_MR:
		case IB_WR_RDMA_WRITE:
		case IB_WR_RDMA_READ:
		case IB_WR_RDMA_READ_WITH_INV:
		@@ -770,24 +761,26 @@ out:
		return ret;
		}

		static void rds_iw_build_send_fastreg(struct rds_iw_device rds_iwdev, struct rds_iw_connection ic, struct rds_iw_send_work *send, int nent, int len, u64 sg_addr)
		static int rds_iw_build_send_reg(struct rds_iw_send_work *send,
		struct scatterlist *sg,
		int sg_nents)
		{
		BUG_ON(nent > send->s_page_list->max_page_list_len);
		/*
		* Perform a WR for the fast_reg_mr. Each individual page
		* in the sg list is added to the fast reg page list and placed
		* inside the fast_reg_mr WR.
		*/
		send->s_fast_reg_wr.wr.opcode = IB_WR_FAST_REG_MR;
		send->s_fast_reg_wr.length = len;
		send->s_fast_reg_wr.rkey = send->s_mr->rkey;
		send->s_fast_reg_wr.page_list = send->s_page_list;
		send->s_fast_reg_wr.page_list_len = nent;
		send->s_fast_reg_wr.page_shift = PAGE_SHIFT;
		send->s_fast_reg_wr.access_flags = IB_ACCESS_REMOTE_WRITE;
		send->s_fast_reg_wr.iova_start = sg_addr;
		int n;

		n = ib_map_mr_sg(send->s_mr, sg, sg_nents, PAGE_SIZE);
		if (unlikely(n != sg_nents))
		return n < 0 ? n : -EINVAL;

		send->s_reg_wr.wr.opcode = IB_WR_REG_MR;
		send->s_reg_wr.wr.wr_id = 0;
		send->s_reg_wr.wr.num_sge = 0;
		send->s_reg_wr.mr = send->s_mr;
		send->s_reg_wr.key = send->s_mr->rkey;
		send->s_reg_wr.access = IB_ACCESS_REMOTE_WRITE;

		ib_update_fast_reg_key(send->s_mr, send->s_remap_count++);

		return 0;
		}

		int rds_iw_xmit_rdma(struct rds_connection conn, struct rm_rdma_op op)
		@@ -808,6 +801,7 @@ int rds_iw_xmit_rdma(struct rds_connection conn, struct rm_rdma_op op)
		int sent;
		int ret;
		int num_sge;
		int sg_nents;

		rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client);

		@@ -861,6 +855,7 @@ int rds_iw_xmit_rdma(struct rds_connection conn, struct rm_rdma_op op)
		scat = &op->op_sg[0];
		sent = 0;
		num_sge = op->op_count;
		sg_nents = 0;

		for (i = 0; i < work_alloc && scat != &op->op_sg[op->op_count]; i++) {
		send->s_rdma_wr.wr.send_flags = 0;
		@@ -904,7 +899,7 @@ int rds_iw_xmit_rdma(struct rds_connection conn, struct rm_rdma_op op)
		len = ib_sg_dma_len(ic->i_cm_id->device, scat);

		if (send->s_rdma_wr.wr.opcode == IB_WR_RDMA_READ_WITH_INV)
		send->s_page_list->page_list[j] = ib_sg_dma_address(ic->i_cm_id->device, scat);
		sg_nents++;
		else {
		send->s_sge[j].addr = ib_sg_dma_address(ic->i_cm_id->device, scat);
		send->s_sge[j].length = len;
		@@ -951,8 +946,12 @@ int rds_iw_xmit_rdma(struct rds_connection conn, struct rm_rdma_op op)
		* fastreg_mr (or possibly a dma_mr)
		*/
		if (!op->op_write) {
		rds_iw_build_send_fastreg(rds_iwdev, ic, &ic->i_sends[fr_pos],
		op->op_count, sent, conn->c_xmit_rm->m_rs->rs_user_addr);
		ret = rds_iw_build_send_reg(&ic->i_sends[fr_pos],
		&op->op_sg[0], sg_nents);
		if (ret) {
		printk(KERN_WARNING "RDS/IW: failed to reg send mem\n");
		goto out;
		}
		work_alloc++;
		}

Admin message