Commit f16b6e8d authored by NeilBrown's avatar NeilBrown Committed by J. Bruce Fields
Browse files

sunrpc/cache: allow threads to block while waiting for cache update. 



The current practice of waiting for cache updates by queueing the
whole request to be retried has (at least) two problems.

1/ With NFSv4, requests can be quite complex and re-trying a whole
  request when a later part fails should only be a last-resort, not a
  normal practice.

2/ Large requests, and in particular any 'write' request, will not be
  queued by the current code and doing so would be undesirable.

In many cases only a very sort wait is needed before the cache gets
valid data.

So, providing the underlying transport permits it by setting
 ->thread_wait,
arrange to wait briefly for an upcall to be completed (as reflected in
the clearing of CACHE_PENDING).
If the short wait was not long enough and CACHE_PENDING is still set,
fall back on the old approach.

The 'thread_wait' value is set to 5 seconds when there are spare
threads, and 1 second when there are no spare threads.

These values are probably much higher than needed, but will ensure
some forward progress.

Note that as we only request an update for a non-valid item, and as
non-valid items are updated in place it is extremely unlikely that
cache_check will return -ETIMEDOUT.  Normally cache_defer_req will
sleep for a short while and then find that the item is_valid.

Signed-off-by: default avatarNeilBrown <neilb@suse.de>
Signed-off-by: default avatarJ. Bruce Fields <bfields@redhat.com>
parent c5b29f88

include/linux/sunrpc/cache.h

+3 −0
Original line number Original line Diff line number Diff line
@@ -125,6 +125,9 @@ struct cache_detail { 
 */

 */

struct cache_req {

struct cache_req {

	struct cache_deferred_req *(*defer)(struct cache_req *req);

	struct cache_deferred_req *(*defer)(struct cache_req *req);

	int thread_wait;  /* How long (jiffies) we can block the

			   * current thread to wait for updates.

			   */

};

};

/* this must be embedded in a deferred_request that is being

/* this must be embedded in a deferred_request that is being

 * delayed awaiting cache-fill

 * delayed awaiting cache-fill

net/sunrpc/cache.c

+58 −1
Original line number Original line Diff line number Diff line
@@ -509,10 +509,22 @@ static LIST_HEAD(cache_defer_list); 
static struct list_head cache_defer_hash[DFR_HASHSIZE];

static struct list_head cache_defer_hash[DFR_HASHSIZE];

static int cache_defer_cnt;

static int cache_defer_cnt;





struct thread_deferred_req {

	struct cache_deferred_req handle;

	struct completion completion;

};

static void cache_restart_thread(struct cache_deferred_req *dreq, int too_many)

{

	struct thread_deferred_req *dr =

		container_of(dreq, struct thread_deferred_req, handle);

	complete(&dr->completion);

}



static int cache_defer_req(struct cache_req *req, struct cache_head *item)

static int cache_defer_req(struct cache_req *req, struct cache_head *item)

{

{

	struct cache_deferred_req *dreq, *discard;

	struct cache_deferred_req *dreq, *discard;

	int hash = DFR_HASH(item);

	int hash = DFR_HASH(item);

	struct thread_deferred_req sleeper;





	if (cache_defer_cnt >= DFR_MAX) {

	if (cache_defer_cnt >= DFR_MAX) {

		/* too much in the cache, randomly drop this one,

		/* too much in the cache, randomly drop this one,
@@ -521,7 +533,15 @@ static int cache_defer_req(struct cache_req *req, struct cache_head *item) 
		if (net_random()&1)

		if (net_random()&1)

			return -ENOMEM;

			return -ENOMEM;

	}

	}

	if (req->thread_wait) {

		dreq = &sleeper.handle;

		sleeper.completion =

			COMPLETION_INITIALIZER_ONSTACK(sleeper.completion);

		dreq->revisit = cache_restart_thread;

	} else

		dreq = req->defer(req);

		dreq = req->defer(req);



 retry:

	if (dreq == NULL)

	if (dreq == NULL)

		return -ENOMEM;

		return -ENOMEM;
@@ -555,6 +575,43 @@ static int cache_defer_req(struct cache_req *req, struct cache_head *item) 
		cache_revisit_request(item);

		cache_revisit_request(item);

		return -EAGAIN;

		return -EAGAIN;

	}

	}



	if (dreq == &sleeper.handle) {

		if (wait_for_completion_interruptible_timeout(

			    &sleeper.completion, req->thread_wait) <= 0) {

			/* The completion wasn't completed, so we need

			 * to clean up

			 */

			spin_lock(&cache_defer_lock);

			if (!list_empty(&sleeper.handle.hash)) {

				list_del_init(&sleeper.handle.recent);

				list_del_init(&sleeper.handle.hash);

				cache_defer_cnt--;

				spin_unlock(&cache_defer_lock);

			} else {

				/* cache_revisit_request already removed

				 * this from the hash table, but hasn't

				 * called ->revisit yet.  It will very soon

				 * and we need to wait for it.

				 */

				spin_unlock(&cache_defer_lock);

				wait_for_completion(&sleeper.completion);

			}

		}

		if (test_bit(CACHE_PENDING, &item->flags)) {

			/* item is still pending, try request

			 * deferral

			 */

			dreq = req->defer(req);

			goto retry;

		}

		/* only return success if we actually deferred the

		 * request.  In this case we waited until it was

		 * answered so no deferral has happened - rather

		 * an answer already exists.

		 */

		return -EEXIST;

	}

	return 0;

	return 0;

}

}

net/sunrpc/svc_xprt.c

+11 −0
Original line number Original line Diff line number Diff line
@@ -651,6 +651,11 @@ int svc_recv(struct svc_rqst *rqstp, long timeout) 
	if (signalled() || kthread_should_stop())

	if (signalled() || kthread_should_stop())

		return -EINTR;

		return -EINTR;





	/* Normally we will wait up to 5 seconds for any required

	 * cache information to be provided.

	 */

	rqstp->rq_chandle.thread_wait = 5*HZ;



	spin_lock_bh(&pool->sp_lock);

	spin_lock_bh(&pool->sp_lock);

	xprt = svc_xprt_dequeue(pool);

	xprt = svc_xprt_dequeue(pool);

	if (xprt) {

	if (xprt) {
@@ -658,6 +663,12 @@ int svc_recv(struct svc_rqst *rqstp, long timeout) 
		svc_xprt_get(xprt);

		svc_xprt_get(xprt);

		rqstp->rq_reserved = serv->sv_max_mesg;

		rqstp->rq_reserved = serv->sv_max_mesg;

		atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);

		atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);



		/* As there is a shortage of threads and this request

		 * had to be queue, don't allow the thread to wait so

		 * long for cache updates.

		 */

		rqstp->rq_chandle.thread_wait = 1*HZ;

	} else {

	} else {

		/* No data pending. Go to sleep */

		/* No data pending. Go to sleep */

		svc_thread_enqueue(pool, rqstp);

		svc_thread_enqueue(pool, rqstp);

CRA Git | Maintained and supported by SUSTech CRA and CCSE