Commit 661373b1 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Trond Myklebust
Browse files

pnfs: factor GETDEVICEINFO implementations 



Add support to the common pNFS core to issue GETDEVICEINFO calls on
a device ID cache miss.  The code is taken from the well debugged
file layout implementation and calls out to the layoutdriver through
a new alloc_deviceid_node method.  The calling conventions for
nfs4_find_get_deviceid are changed so that all information needed to
send a GETDEVICEINFO request is passed to the common code.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarTrond Myklebust <trond.myklebust@primarydata.com>
parent 848746bd

fs/nfs/filelayout/filelayout.c

+19 −10
Original line number Diff line number Diff line
@@ -649,14 +649,11 @@ filelayout_check_layout(struct pnfs_layout_hdr *lo, 
	}



	/* find and reference the deviceid */

	d = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode)->pnfs_curr_ld,

				   NFS_SERVER(lo->plh_inode)->nfs_client, id);

	if (d == NULL) {

		dsaddr = filelayout_get_device_info(lo->plh_inode, id,

	d = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode), id,

			lo->plh_lc_cred, gfp_flags);

		if (dsaddr == NULL)

	if (d == NULL)

		goto out;

	} else



	dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);

	/* Found deviceid is unavailable */

	if (filelayout_test_devid_unavailable(&dsaddr->id_node))
@@ -1371,6 +1368,17 @@ out: 
	cinfo->ds->ncommitting = 0;

	return PNFS_ATTEMPTED;

}

static struct nfs4_deviceid_node *

filelayout_alloc_deviceid_node(struct nfs_server *server,

		struct pnfs_device *pdev, gfp_t gfp_flags)

{

	struct nfs4_file_layout_dsaddr *dsaddr;



	dsaddr = nfs4_fl_alloc_deviceid_node(server, pdev, gfp_flags);

	if (!dsaddr)

		return NULL;

	return &dsaddr->id_node;

}



static void

filelayout_free_deveiceid_node(struct nfs4_deviceid_node *d)
@@ -1423,6 +1431,7 @@ static struct pnfs_layoutdriver_type filelayout_type = { 
	.commit_pagelist	= filelayout_commit_pagelist,

	.read_pagelist		= filelayout_read_pagelist,

	.write_pagelist		= filelayout_write_pagelist,

	.alloc_deviceid_node	= filelayout_alloc_deviceid_node,

	.free_deviceid_node	= filelayout_free_deveiceid_node,

};

fs/nfs/filelayout/filelayout.h

+4 −3
Original line number Diff line number Diff line
@@ -147,10 +147,11 @@ u32 nfs4_fl_calc_j_index(struct pnfs_layout_segment *lseg, loff_t offset); 
u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, u32 j);

struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,

					u32 ds_idx);



extern struct nfs4_file_layout_dsaddr *

nfs4_fl_alloc_deviceid_node(struct nfs_server *server,

	struct pnfs_device *pdev, gfp_t gfp_flags);

extern void nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr);

extern void nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr);

struct nfs4_file_layout_dsaddr *

filelayout_get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id,

		struct rpc_cred *cred, gfp_t gfp_flags);



#endif /* FS_NFS_NFS4FILELAYOUT_H */

fs/nfs/filelayout/filelayoutdev.c

+5 −103
Original line number Diff line number Diff line
@@ -484,8 +484,9 @@ out_err: 
}



/* Decode opaque device data and return the result */

static struct nfs4_file_layout_dsaddr*

decode_device(struct inode *ino, struct pnfs_device *pdev, gfp_t gfp_flags)

struct nfs4_file_layout_dsaddr *

nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,

		gfp_t gfp_flags)

{

	int i;

	u32 cnt, num;
@@ -570,10 +571,7 @@ decode_device(struct inode *ino, struct pnfs_device *pdev, gfp_t gfp_flags) 
	dsaddr->stripe_indices = stripe_indices;

	stripe_indices = NULL;

	dsaddr->ds_num = num;

	nfs4_init_deviceid_node(&dsaddr->id_node,

				NFS_SERVER(ino)->pnfs_curr_ld,

				NFS_SERVER(ino)->nfs_client,

				&pdev->dev_id);

	nfs4_init_deviceid_node(&dsaddr->id_node, server, &pdev->dev_id);



	INIT_LIST_HEAD(&dsaddrs);
@@ -587,7 +585,7 @@ decode_device(struct inode *ino, struct pnfs_device *pdev, gfp_t gfp_flags) 


		mp_count = be32_to_cpup(p); /* multipath count */

		for (j = 0; j < mp_count; j++) {

			da = decode_ds_addr(NFS_SERVER(ino)->nfs_client->cl_net,

			da = decode_ds_addr(server->nfs_client->cl_net,

					    &stream, gfp_flags);

			if (da)

				list_add_tail(&da->da_node, &dsaddrs);
@@ -637,102 +635,6 @@ out_err: 
	return NULL;

}



/*

 * Decode the opaque device specified in 'dev' and add it to the cache of

 * available devices.

 */

static struct nfs4_file_layout_dsaddr *

decode_and_add_device(struct inode *inode, struct pnfs_device *dev, gfp_t gfp_flags)

{

	struct nfs4_deviceid_node *d;

	struct nfs4_file_layout_dsaddr *n, *new;



	new = decode_device(inode, dev, gfp_flags);

	if (!new) {

		printk(KERN_WARNING "NFS: %s: Could not decode or add device\n",

			__func__);

		return NULL;

	}



	d = nfs4_insert_deviceid_node(&new->id_node);

	n = container_of(d, struct nfs4_file_layout_dsaddr, id_node);

	if (n != new) {

		nfs4_fl_free_deviceid(new);

		return n;

	}



	return new;

}



/*

 * Retrieve the information for dev_id, add it to the list

 * of available devices, and return it.

 */

struct nfs4_file_layout_dsaddr *

filelayout_get_device_info(struct inode *inode,

		struct nfs4_deviceid *dev_id,

		struct rpc_cred *cred,

		gfp_t gfp_flags)

{

	struct pnfs_device *pdev = NULL;

	u32 max_resp_sz;

	int max_pages;

	struct page **pages = NULL;

	struct nfs4_file_layout_dsaddr *dsaddr = NULL;

	int rc, i;

	struct nfs_server *server = NFS_SERVER(inode);



	/*

	 * Use the session max response size as the basis for setting

	 * GETDEVICEINFO's maxcount

	 */

	max_resp_sz = server->nfs_client->cl_session->fc_attrs.max_resp_sz;

	max_pages = nfs_page_array_len(0, max_resp_sz);

	dprintk("%s inode %p max_resp_sz %u max_pages %d\n",

		__func__, inode, max_resp_sz, max_pages);



	pdev = kzalloc(sizeof(struct pnfs_device), gfp_flags);

	if (pdev == NULL)

		return NULL;



	pages = kcalloc(max_pages, sizeof(struct page *), gfp_flags);

	if (pages == NULL) {

		kfree(pdev);

		return NULL;

	}

	for (i = 0; i < max_pages; i++) {

		pages[i] = alloc_page(gfp_flags);

		if (!pages[i])

			goto out_free;

	}



	memcpy(&pdev->dev_id, dev_id, sizeof(*dev_id));

	pdev->layout_type = LAYOUT_NFSV4_1_FILES;

	pdev->pages = pages;

	pdev->pgbase = 0;

	pdev->pglen = max_resp_sz;

	pdev->mincount = 0;

	pdev->maxcount = max_resp_sz - nfs41_maxgetdevinfo_overhead;



	rc = nfs4_proc_getdeviceinfo(server, pdev, cred);

	dprintk("%s getdevice info returns %d\n", __func__, rc);

	if (rc)

		goto out_free;



	/*

	 * Found new device, need to decode it and then add it to the

	 * list of known devices for this mountpoint.

	 */

	dsaddr = decode_and_add_device(inode, pdev, gfp_flags);

out_free:

	for (i = 0; i < max_pages; i++)

		__free_page(pages[i]);

	kfree(pages);

	kfree(pdev);

	dprintk("<-- %s dsaddr %p\n", __func__, dsaddr);

	return dsaddr;

}



void

nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)

{

fs/nfs/objlayout/objio_osd.c

+41 −72
Original line number Diff line number Diff line
@@ -60,52 +60,6 @@ objio_free_deviceid_node(struct nfs4_deviceid_node *d) 
	kfree(de);

}



static struct objio_dev_ent *_dev_list_find(const struct nfs_server *nfss,

	const struct nfs4_deviceid *d_id)

{

	struct nfs4_deviceid_node *d;

	struct objio_dev_ent *de;



	d = nfs4_find_get_deviceid(nfss->pnfs_curr_ld, nfss->nfs_client, d_id);

	if (!d)

		return NULL;



	de = container_of(d, struct objio_dev_ent, id_node);

	return de;

}



static struct objio_dev_ent *

_dev_list_add(const struct nfs_server *nfss,

	const struct nfs4_deviceid *d_id, struct osd_dev *od,

	gfp_t gfp_flags)

{

	struct nfs4_deviceid_node *d;

	struct objio_dev_ent *de = kzalloc(sizeof(*de), gfp_flags);

	struct objio_dev_ent *n;



	if (!de) {

		dprintk("%s: -ENOMEM od=%p\n", __func__, od);

		return NULL;

	}



	dprintk("%s: Adding od=%p\n", __func__, od);

	nfs4_init_deviceid_node(&de->id_node,

				nfss->pnfs_curr_ld,

				nfss->nfs_client,

				d_id);

	de->od.od = od;



	d = nfs4_insert_deviceid_node(&de->id_node);

	n = container_of(d, struct objio_dev_ent, id_node);

	if (n != de) {

		dprintk("%s: Race with other n->od=%p\n", __func__, n->od.od);

		objio_free_deviceid_node(&de->id_node);

		de = n;

	}



	return de;

}



struct objio_segment {

	struct pnfs_layout_segment lseg;
@@ -130,29 +84,24 @@ struct objio_state { 


/* Send and wait for a get_device_info of devices in the layout,

   then look them up with the osd_initiator library */

static int objio_devices_lookup(struct pnfs_layout_hdr *pnfslay,

	struct objio_segment *objio_seg, unsigned c, struct nfs4_deviceid *d_id,

struct nfs4_deviceid_node *

objio_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,

			gfp_t gfp_flags)

{

	struct pnfs_osd_deviceaddr *deviceaddr;

	struct objio_dev_ent *ode;

	struct objio_dev_ent *ode = NULL;

	struct osd_dev *od;

	struct osd_dev_info odi;

	bool retry_flag = true;

	u32 *p;

	int err;



	ode = _dev_list_find(NFS_SERVER(pnfslay->plh_inode), d_id);

	if (ode) {

		objio_seg->oc.ods[c] = &ode->od; /* must use container_of */

		return 0;

	}

	deviceaddr = kzalloc(sizeof(*deviceaddr), gfp_flags);

	if (!deviceaddr)

		return NULL;



	err = objlayout_get_deviceinfo(pnfslay, d_id, &deviceaddr, gfp_flags);

	if (unlikely(err)) {

		dprintk("%s: objlayout_get_deviceinfo dev(%llx:%llx) =>%d\n",

			__func__, _DEVID_LO(d_id), _DEVID_HI(d_id), err);

		return err;

	}

	p = page_address(pdev->pages[0]);

	pnfs_osd_xdr_decode_deviceaddr(deviceaddr, p);



	odi.systemid_len = deviceaddr->oda_systemid.len;

	if (odi.systemid_len > sizeof(odi.systemid)) {
@@ -188,14 +137,24 @@ retry_lookup: 
		goto out;

	}



	ode = _dev_list_add(NFS_SERVER(pnfslay->plh_inode), d_id, od,

			    gfp_flags);

	objio_seg->oc.ods[c] = &ode->od; /* must use container_of */

	dprintk("Adding new dev_id(%llx:%llx)\n",

		_DEVID_LO(d_id), _DEVID_HI(d_id));

		_DEVID_LO(&pdev->dev_id), _DEVID_HI(&pdev->dev_id));



	ode = kzalloc(sizeof(*ode), gfp_flags);

	if (!ode) {

		dprintk("%s: -ENOMEM od=%p\n", __func__, od);

		goto out;

	}



	nfs4_init_deviceid_node(&ode->id_node, server, &pdev->dev_id);

	kfree(deviceaddr);



	ode->od.od = od;

	return &ode->id_node;



out:

	objlayout_put_deviceinfo(deviceaddr);

	return err;

	kfree(deviceaddr);

	return NULL;

}



static void copy_single_comp(struct ore_components *oc, unsigned c,
@@ -254,6 +213,7 @@ int objio_alloc_lseg(struct pnfs_layout_segment **outp, 
	struct xdr_stream *xdr,

	gfp_t gfp_flags)

{

	struct nfs_server *server = NFS_SERVER(pnfslay->plh_inode);

	struct objio_segment *objio_seg;

	struct pnfs_osd_xdr_decode_layout_iter iter;

	struct pnfs_osd_layout layout;
@@ -283,13 +243,21 @@ int objio_alloc_lseg(struct pnfs_layout_segment **outp, 
	objio_seg->oc.first_dev = layout.olo_comps_index;

	cur_comp = 0;

	while (pnfs_osd_xdr_decode_layout_comp(&src_comp, &iter, xdr, &err)) {

		struct nfs4_deviceid_node *d;

		struct objio_dev_ent *ode;



		copy_single_comp(&objio_seg->oc, cur_comp, &src_comp);

		err = objio_devices_lookup(pnfslay, objio_seg, cur_comp,



		d = nfs4_find_get_deviceid(server,

				&src_comp.oc_object_id.oid_device_id,

					   gfp_flags);

		if (err)

				pnfslay->plh_lc_cred, gfp_flags);

		if (!d) {

			err = -ENXIO;

			goto err;

		++cur_comp;

		}



		ode = container_of(d, struct objio_dev_ent, id_node);

		objio_seg->oc.ods[cur_comp++] = &ode->od;

	}

	/* pnfs_osd_xdr_decode_layout_comp returns false on error */

	if (unlikely(err))
@@ -653,6 +621,7 @@ static struct pnfs_layoutdriver_type objlayout_type = { 
	.flags                   = PNFS_LAYOUTRET_ON_SETATTR |

				   PNFS_LAYOUTRET_ON_ERROR,



	.max_deviceinfo_size	 = PAGE_SIZE,

	.owner		       	 = THIS_MODULE,

	.alloc_layout_hdr        = objlayout_alloc_layout_hdr,

	.free_layout_hdr         = objlayout_free_layout_hdr,

fs/nfs/objlayout/objlayout.c

+0 −70
Original line number Diff line number Diff line
@@ -574,76 +574,6 @@ loop_done: 
	dprintk("%s: Return\n", __func__);

}





/*

 * Get Device Info API for io engines

 */

struct objlayout_deviceinfo {

	struct page *page;

	struct pnfs_osd_deviceaddr da; /* This must be last */

};



/* Initialize and call nfs_getdeviceinfo, then decode and return a

 * "struct pnfs_osd_deviceaddr *" Eventually objlayout_put_deviceinfo()

 * should be called.

 */

int objlayout_get_deviceinfo(struct pnfs_layout_hdr *pnfslay,

	struct nfs4_deviceid *d_id, struct pnfs_osd_deviceaddr **deviceaddr,

	gfp_t gfp_flags)

{

	struct objlayout_deviceinfo *odi;

	struct pnfs_device pd;

	struct page *page, **pages;

	u32 *p;

	int err;



	page = alloc_page(gfp_flags);

	if (!page)

		return -ENOMEM;



	pages = &page;

	pd.pages = pages;



	memcpy(&pd.dev_id, d_id, sizeof(*d_id));

	pd.layout_type = LAYOUT_OSD2_OBJECTS;

	pd.pages = &page;

	pd.pgbase = 0;

	pd.pglen = PAGE_SIZE;

	pd.mincount = 0;

	pd.maxcount = PAGE_SIZE;



	err = nfs4_proc_getdeviceinfo(NFS_SERVER(pnfslay->plh_inode), &pd,

			pnfslay->plh_lc_cred);

	dprintk("%s nfs_getdeviceinfo returned %d\n", __func__, err);

	if (err)

		goto err_out;



	p = page_address(page);

	odi = kzalloc(sizeof(*odi), gfp_flags);

	if (!odi) {

		err = -ENOMEM;

		goto err_out;

	}

	pnfs_osd_xdr_decode_deviceaddr(&odi->da, p);

	odi->page = page;

	*deviceaddr = &odi->da;

	return 0;



err_out:

	__free_page(page);

	return err;

}



void objlayout_put_deviceinfo(struct pnfs_osd_deviceaddr *deviceaddr)

{

	struct objlayout_deviceinfo *odi = container_of(deviceaddr,

						struct objlayout_deviceinfo,

						da);



	__free_page(odi->page);

	kfree(odi);

}



enum {

	OBJLAYOUT_MAX_URI_LEN = 256, OBJLAYOUT_MAX_OSDNAME_LEN = 64,

	OBJLAYOUT_MAX_SYSID_HEX_LEN = OSD_SYSTEMID_LEN * 2 + 1,

CRA Git | Maintained and supported by SUSTech CRA and CCSE