Commit 8dd37758 authored by Trond Myklebust's avatar Trond Myklebust
Browse files

NFSv4.1: Clean ups and bugfixes for the pNFS read/writeback/commit code 



Move more pnfs-isms out of the generic commit code.

Bugfixes:

- filelayout_scan_commit_lists doesn't need to get/put the lseg.
  In fact since it is run under the inode->i_lock, the lseg_put()
  can deadlock.

- Ensure that we distinguish between what needs to be done for
  commit-to-data server and what needs to be done for commit-to-MDS
  using the new flag PG_COMMIT_TO_DS. Otherwise we may end up calling
  put_lseg() on a bucket for a struct nfs_page that got written
  through the MDS.

- Fix a case where we were using list_del() on an nfs_page->wb_list
  instead of list_del_init().

- filelayout_initiate_commit needs to call filelayout_commit_release
  on error instead of the mds_ops->rpc_release(). Otherwise it won't
  clear the commit lock.

Cleanups:

- Let the files layout manage the commit lists for the pNFS case.
  Don't expose stuff like pnfs_choose_commit_list, and the fact
  that the commit buckets hold references to the layout segment
  in common code.

- Cast out the put_lseg() calls for the struct nfs_read/write_data->lseg
  into the pNFS layer from whence they came.

- Let the pNFS layer manage the NFS_INO_PNFS_COMMIT bit.

Signed-off-by: default avatarTrond Myklebust <Trond.Myklebust@netapp.com>
Cc: Fred Isaman <iisaman@netapp.com>
parent 95a13f7b

fs/nfs/internal.h

+2 −2
Original line number Diff line number Diff line
@@ -308,8 +308,6 @@ extern void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio); 
extern void nfs_readdata_release(struct nfs_read_data *rdata);



/* write.c */

extern int nfs_scan_commit_list(struct list_head *src, struct list_head *dst,

				int max);

extern int nfs_generic_flush(struct nfs_pageio_descriptor *desc,

		struct list_head *head);

extern void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
@@ -334,6 +332,8 @@ void nfs_retry_commit(struct list_head *page_list, 
void nfs_commit_clear_lock(struct nfs_inode *nfsi);

void nfs_commitdata_release(void *data);

void nfs_commit_release_pages(struct nfs_write_data *data);

void nfs_request_add_commit_list(struct nfs_page *req, struct list_head *head);

void nfs_request_remove_commit_list(struct nfs_page *req);



#ifdef CONFIG_MIGRATION

extern int nfs_migrate_page(struct address_space *,

fs/nfs/nfs4filelayout.c

+62 −20
Original line number Diff line number Diff line
@@ -224,6 +224,7 @@ static void filelayout_read_release(void *data) 
{

	struct nfs_read_data *rdata = (struct nfs_read_data *)data;



	put_lseg(rdata->lseg);

	rdata->mds_ops->rpc_release(data);

}
@@ -310,6 +311,7 @@ static void filelayout_write_release(void *data) 
{

	struct nfs_write_data *wdata = (struct nfs_write_data *)data;



	put_lseg(wdata->lseg);

	wdata->mds_ops->rpc_release(data);

}
@@ -320,6 +322,7 @@ static void filelayout_commit_release(void *data) 
	nfs_commit_release_pages(wdata);

	if (atomic_dec_and_test(&NFS_I(wdata->inode)->commits_outstanding))

		nfs_commit_clear_lock(NFS_I(wdata->inode));

	put_lseg(wdata->lseg);

	nfs_commitdata_release(wdata);

}
@@ -779,11 +782,16 @@ static u32 select_bucket_index(struct nfs4_filelayout_segment *fl, u32 j) 


/* The generic layer is about to remove the req from the commit list.

 * If this will make the bucket empty, it will need to put the lseg reference.

 * Note inode lock is held, so we can't do the put here.

 */

static struct pnfs_layout_segment *

filelayout_remove_commit_req(struct nfs_page *req)

static void

filelayout_clear_request_commit(struct nfs_page *req)

{

	struct pnfs_layout_segment *freeme = NULL;

	struct inode *inode = req->wb_context->dentry->d_inode;



	spin_lock(&inode->i_lock);

	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))

		goto out;

	if (list_is_singular(&req->wb_list)) {

		struct inode *inode = req->wb_context->dentry->d_inode;

		struct pnfs_layout_segment *lseg;
@@ -792,11 +800,16 @@ filelayout_remove_commit_req(struct nfs_page *req) 
		 * since there is only one relevant lseg...

		 */

		list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {

			if (lseg->pls_range.iomode == IOMODE_RW)

				return lseg;

			if (lseg->pls_range.iomode == IOMODE_RW) {

				freeme = lseg;

				break;

			}

		}

	return NULL;

	}

out:

	nfs_request_remove_commit_list(req);

	spin_unlock(&inode->i_lock);

	put_lseg(freeme);

}



static struct list_head *
@@ -829,9 +842,20 @@ filelayout_choose_commit_list(struct nfs_page *req, 
		 */

		get_lseg(lseg);

	}

	set_bit(PG_COMMIT_TO_DS, &req->wb_flags);

	return list;

}



static void

filelayout_mark_request_commit(struct nfs_page *req,

		struct pnfs_layout_segment *lseg)

{

	struct list_head *list;



	list = filelayout_choose_commit_list(req, lseg);

	nfs_request_add_commit_list(req, list);

}



static u32 calc_ds_index_from_commit(struct pnfs_layout_segment *lseg, u32 i)

{

	struct nfs4_filelayout_segment *flseg = FILELAYOUT_LSEG(lseg);
@@ -872,7 +896,7 @@ static int filelayout_initiate_commit(struct nfs_write_data *data, int how) 
		set_bit(lo_fail_bit(IOMODE_RW), &lseg->pls_layout->plh_flags);

		set_bit(lo_fail_bit(IOMODE_READ), &lseg->pls_layout->plh_flags);

		prepare_to_resend_writes(data);

		data->mds_ops->rpc_release(data);

		filelayout_commit_release(data);

		return -EAGAIN;

	}

	dprintk("%s ino %lu, how %d\n", __func__, data->inode->i_ino, how);
@@ -895,7 +919,7 @@ find_only_write_lseg_locked(struct inode *inode) 


	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list)

		if (lseg->pls_range.iomode == IOMODE_RW)

			return get_lseg(lseg);

			return lseg;

	return NULL;

}
@@ -905,10 +929,33 @@ static struct pnfs_layout_segment *find_only_write_lseg(struct inode *inode) 


	spin_lock(&inode->i_lock);

	rv = find_only_write_lseg_locked(inode);

	if (rv)

		get_lseg(rv);

	spin_unlock(&inode->i_lock);

	return rv;

}



static int

filelayout_scan_ds_commit_list(struct nfs4_fl_commit_bucket *bucket, int max)

{

	struct list_head *src = &bucket->written;

	struct list_head *dst = &bucket->committing;

	struct nfs_page *req, *tmp;

	int ret = 0;



	list_for_each_entry_safe(req, tmp, src, wb_list) {

		if (!nfs_lock_request(req))

			continue;

		nfs_request_remove_commit_list(req);

		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);

		nfs_list_add_request(req, dst);

		ret++;

		if (ret == max)

			break;

	}

	return ret;

}



/* Move reqs from written to committing lists, returning count of number moved.

 * Note called with i_lock held.

 */
@@ -920,21 +967,16 @@ static int filelayout_scan_commit_lists(struct inode *inode, int max) 


	lseg = find_only_write_lseg_locked(inode);

	if (!lseg)

		return 0;

		goto out_done;

	fl = FILELAYOUT_LSEG(lseg);

	if (fl->commit_through_mds)

		goto out_put;

	for (i = 0; i < fl->number_of_buckets; i++) {

		if (list_empty(&fl->commit_buckets[i].written))

			continue;

		cnt = nfs_scan_commit_list(&fl->commit_buckets[i].written,

					   &fl->commit_buckets[i].committing,

					   max);

		goto out_done;

	for (i = 0; i < fl->number_of_buckets && max != 0; i++) {

		cnt = filelayout_scan_ds_commit_list(&fl->commit_buckets[i], max);

		max -= cnt;

		rv += cnt;

	}

out_put:

	put_lseg(lseg);

out_done:

	return rv;

}
@@ -1033,8 +1075,8 @@ static struct pnfs_layoutdriver_type filelayout_type = { 
	.free_lseg		= filelayout_free_lseg,

	.pg_read_ops		= &filelayout_pg_read_ops,

	.pg_write_ops		= &filelayout_pg_write_ops,

	.choose_commit_list	= filelayout_choose_commit_list,

	.remove_commit_req	= filelayout_remove_commit_req,

	.mark_request_commit	= filelayout_mark_request_commit,

	.clear_request_commit	= filelayout_clear_request_commit,

	.scan_commit_lists	= filelayout_scan_commit_lists,

	.commit_pagelist	= filelayout_commit_pagelist,

	.read_pagelist		= filelayout_read_pagelist,

fs/nfs/pnfs.c

+3 −0
Original line number Diff line number Diff line
@@ -1210,6 +1210,7 @@ void pnfs_ld_write_done(struct nfs_write_data *data) 
		}

		data->task.tk_status = pnfs_write_done_resend_to_mds(data->inode, &data->pages);

	}

	put_lseg(data->lseg);

	data->mds_ops->rpc_release(data);

}

EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
@@ -1223,6 +1224,7 @@ pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 
		nfs_list_add_request(data->req, &desc->pg_list);

	nfs_pageio_reset_write_mds(desc);

	desc->pg_recoalesce = 1;

	put_lseg(data->lseg);

	nfs_writedata_release(data);

}
@@ -1323,6 +1325,7 @@ void pnfs_ld_read_done(struct nfs_read_data *data) 
		data->mds_ops->rpc_call_done(&data->task, data);

	} else

		pnfs_ld_handle_read_error(data);

	put_lseg(data->lseg);

	data->mds_ops->rpc_release(data);

}

EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

fs/nfs/pnfs.h

+28 −27
Original line number Diff line number Diff line
@@ -94,9 +94,9 @@ struct pnfs_layoutdriver_type { 
	const struct nfs_pageio_ops *pg_read_ops;

	const struct nfs_pageio_ops *pg_write_ops;



	struct list_head * (*choose_commit_list) (struct nfs_page *req,

	void (*mark_request_commit) (struct nfs_page *req,

					struct pnfs_layout_segment *lseg);

	struct pnfs_layout_segment *(*remove_commit_req) (struct nfs_page *req);

	void (*clear_request_commit) (struct nfs_page *req);

	int (*scan_commit_lists) (struct inode *inode, int max);

	int (*commit_pagelist)(struct inode *inode, struct list_head *mds_pages, int how);
@@ -269,39 +269,42 @@ pnfs_commit_list(struct inode *inode, struct list_head *mds_pages, int how) 
	return NFS_SERVER(inode)->pnfs_curr_ld->commit_pagelist(inode, mds_pages, how);

}



static inline struct list_head *

pnfs_choose_commit_list(struct nfs_page *req, struct pnfs_layout_segment *lseg)

static inline bool

pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg)

{

	struct inode *inode = req->wb_context->dentry->d_inode;

	struct list_head *rv;

	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;



	if (lseg && NFS_SERVER(inode)->pnfs_curr_ld->choose_commit_list)

		rv = NFS_SERVER(inode)->pnfs_curr_ld->choose_commit_list(req, lseg);

	else

		rv = &NFS_I(inode)->commit_list;

	return rv;

	if (lseg == NULL || ld->mark_request_commit == NULL)

		return false;

	ld->mark_request_commit(req, lseg);

	return true;

}



static inline struct pnfs_layout_segment *

static inline bool

pnfs_clear_request_commit(struct nfs_page *req)

{

	struct inode *inode = req->wb_context->dentry->d_inode;

	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;



	if (NFS_SERVER(inode)->pnfs_curr_ld &&

	    NFS_SERVER(inode)->pnfs_curr_ld->remove_commit_req)

		return NFS_SERVER(inode)->pnfs_curr_ld->remove_commit_req(req);

	else

		return NULL;

	if (ld == NULL || ld->clear_request_commit == NULL)

		return false;

	ld->clear_request_commit(req);

	return true;

}



static inline int

pnfs_scan_commit_lists(struct inode *inode, int max)

{

	if (NFS_SERVER(inode)->pnfs_curr_ld &&

	    NFS_SERVER(inode)->pnfs_curr_ld->scan_commit_lists)

		return NFS_SERVER(inode)->pnfs_curr_ld->scan_commit_lists(inode, max);

	else

	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;

	int ret;



	if (ld == NULL || ld->scan_commit_lists == NULL)

		return 0;

	ret = ld->scan_commit_lists(inode, max);

	if (ret != 0)

		set_bit(NFS_INO_PNFS_COMMIT, &NFS_I(inode)->flags);

	return ret;

}



/* Should the pNFS client commit and return the layout upon a setattr */
@@ -403,18 +406,16 @@ pnfs_commit_list(struct inode *inode, struct list_head *mds_pages, int how) 
	return PNFS_NOT_ATTEMPTED;

}



static inline struct list_head *

pnfs_choose_commit_list(struct nfs_page *req, struct pnfs_layout_segment *lseg)

static inline bool

pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg)

{

	struct inode *inode = req->wb_context->dentry->d_inode;



	return &NFS_I(inode)->commit_list;

	return false;

}



static inline struct pnfs_layout_segment *

static inline bool

pnfs_clear_request_commit(struct nfs_page *req)

{

	return NULL;

	return false;

}



static inline int

fs/nfs/read.c

+0 −1
Original line number Diff line number Diff line
@@ -66,7 +66,6 @@ void nfs_readdata_free(struct nfs_read_data *p) 


void nfs_readdata_release(struct nfs_read_data *rdata)

{

	put_lseg(rdata->lseg);

	put_nfs_open_context(rdata->args.context);

	nfs_readdata_free(rdata);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE