Commit 095975da authored by Nick Piggin's avatar Nick Piggin Committed by Linus Torvalds
Browse files

[PATCH] rcu file: use atomic primitives 



Use atomic_inc_not_zero for rcu files instead of special case rcuref.

Signed-off-by: default avatarNick Piggin <npiggin@suse.de>
Cc: "Paul E. McKenney" <paulmck@us.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent a57004e1

Documentation/RCU/rcuref.txt

+40 −47
Original line number Diff line number Diff line 
Refcounter framework for elements of lists/arrays protected by

RCU.

Refcounter design for elements of lists/arrays protected by RCU.



Refcounting on elements of  lists which are protected by traditional

reader/writer spinlocks or semaphores are straight forward as in:
@@ -31,9 +30,9 @@ release_referenced() delete() 


If this list/array is made lock free using rcu as in changing the

write_lock in add() and delete() to spin_lock and changing read_lock

in search_and_reference to rcu_read_lock(), the rcuref_get in

in search_and_reference to rcu_read_lock(), the atomic_get in

search_and_reference could potentially hold reference to an element which

has already been deleted from the list/array.  rcuref_lf_get_rcu takes

has already been deleted from the list/array.  atomic_inc_not_zero takes

care of this scenario. search_and_reference should look as;



1.					2.
@@ -41,7 +40,7 @@ add() search_and_reference() 
{					{

    alloc_object			    rcu_read_lock();

    ...					    search_for_element

	atomic_set(&el->rc, 1);			if (rcuref_inc_lf(&el->rc)) {

    atomic_set(&el->rc, 1);		    if (atomic_inc_not_zero(&el->rc)) {

    write_lock(&list_lock);		        rcu_read_unlock();

					        return FAIL;

    add_element				    }
@@ -52,23 +51,17 @@ add() search_and_reference() 
release_referenced()			delete()

{					{

    ...					    write_lock(&list_lock);

	rcuref_dec(&el->rc, relfunc)	...

    atomic_dec(&el->rc, relfunc)	    ...

    ...					    delete_element

}					    write_unlock(&list_lock);

 					    ...

					if (rcuref_dec_and_test(&el->rc))

					    if (atomic_dec_and_test(&el->rc))

					        call_rcu(&el->head, el_free);

					    ...

					}



Sometimes, reference to the element need to be obtained in the

update (write) stream.  In such cases, rcuref_inc_lf might be an overkill

since the spinlock serialising list updates are held. rcuref_inc

update (write) stream.  In such cases, atomic_inc_not_zero might be an

overkill since the spinlock serialising list updates are held. atomic_inc

is to be used in such cases.

For arches which do not have cmpxchg rcuref_inc_lf

api uses a hashed spinlock implementation and the same hashed spinlock

is acquired in all rcuref_xxx primitives to preserve atomicity.

Note: Use rcuref_inc api only if you need to use rcuref_inc_lf on the

refcounter atleast at one place.  Mixing rcuref_inc and atomic_xxx api

might lead to races. rcuref_inc_lf() must be used in lockfree

RCU critical sections only.

fs/aio.c

+1 −2
Original line number Diff line number Diff line
@@ -29,7 +29,6 @@ 
#include <linux/highmem.h>

#include <linux/workqueue.h>

#include <linux/security.h>

#include <linux/rcuref.h>



#include <asm/kmap_types.h>

#include <asm/uaccess.h>
@@ -514,7 +513,7 @@ static int __aio_put_req(struct kioctx *ctx, struct kiocb *req) 
	/* Must be done under the lock to serialise against cancellation.

	 * Call this aio_fput as it duplicates fput via the fput_work.

	 */

	if (unlikely(rcuref_dec_and_test(&req->ki_filp->f_count))) {

	if (unlikely(atomic_dec_and_test(&req->ki_filp->f_count))) {

		get_ioctx(ctx);

		spin_lock(&fput_lock);

		list_add(&req->ki_list, &fput_head);

fs/file_table.c

+4 −4
Original line number Diff line number Diff line
@@ -117,7 +117,7 @@ EXPORT_SYMBOL(get_empty_filp); 


void fastcall fput(struct file *file)

{

	if (rcuref_dec_and_test(&file->f_count))

	if (atomic_dec_and_test(&file->f_count))

		__fput(file);

}
@@ -166,7 +166,7 @@ struct file fastcall *fget(unsigned int fd) 
	rcu_read_lock();

	file = fcheck_files(files, fd);

	if (file) {

		if (!rcuref_inc_lf(&file->f_count)) {

		if (!atomic_inc_not_zero(&file->f_count)) {

			/* File object ref couldn't be taken */

			rcu_read_unlock();

			return NULL;
@@ -198,7 +198,7 @@ struct file fastcall *fget_light(unsigned int fd, int *fput_needed) 
		rcu_read_lock();

		file = fcheck_files(files, fd);

		if (file) {

			if (rcuref_inc_lf(&file->f_count))

			if (atomic_inc_not_zero(&file->f_count))

				*fput_needed = 1;

			else

				/* Didn't get the reference, someone's freed */
@@ -213,7 +213,7 @@ struct file fastcall *fget_light(unsigned int fd, int *fput_needed) 


void put_filp(struct file *file)

{

	if (rcuref_dec_and_test(&file->f_count)) {

	if (atomic_dec_and_test(&file->f_count)) {

		security_file_free(file);

		file_kill(file);

		file_free(file);

include/linux/fs.h

+1 −2
Original line number Diff line number Diff line
@@ -9,7 +9,6 @@ 
#include <linux/config.h>

#include <linux/limits.h>

#include <linux/ioctl.h>

#include <linux/rcuref.h>



/*

 * It's silly to have NR_OPEN bigger than NR_FILE, but you can change
@@ -653,7 +652,7 @@ extern spinlock_t files_lock; 
#define file_list_lock() spin_lock(&files_lock);

#define file_list_unlock() spin_unlock(&files_lock);



#define get_file(x)	rcuref_inc(&(x)->f_count)

#define get_file(x)	atomic_inc(&(x)->f_count)

#define file_count(x)	atomic_read(&(x)->f_count)



#define	MAX_NON_LFS	((1UL<<31) - 1)

include/linux/radix-tree.h

+1 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#ifndef _LINUX_RADIX_TREE_H

#define _LINUX_RADIX_TREE_H



#include <linux/sched.h>

#include <linux/preempt.h>

#include <linux/types.h>

CRA Git | Maintained and supported by SUSTech CRA and CCSE