Commit 9ab1544e authored by Lai Jiangshan's avatar Lai Jiangshan Committed by Paul E. McKenney
Browse files

rcu: introduce kfree_rcu() 

Many rcu callbacks functions just call kfree() on the base structure.
These functions are trivial, but their size adds up, and furthermore
when they are used in a kernel module, that module must invoke the
high-latency rcu_barrier() function at module-unload time.

The kfree_rcu() function introduced by this commit addresses this issue.
Rather than encoding a function address in the embedded rcu_head
structure, kfree_rcu() instead encodes the offset of the rcu_head
structure within the base structure.  Because the functions are not
allowed in the low-order 4096 bytes of kernel virtual memory, offsets
up to 4095 bytes can be accommodated.  If the offset is larger than
4095 bytes, a compile-time error will be generated in __kfree_rcu().
If this error is triggered, you can either fall back to use of call_rcu()
or rearrange the structure to position the rcu_head structure into the
first 4096 bytes.

Note that the allowable offset might decrease in the future, for example,
to allow something like kmem_cache_free_rcu().

The new kfree_rcu() function can replace code as follows:

	call_rcu(&p->rcu, simple_kfree_callback);

where "simple_kfree_callback()" might be defined as follows:

	void simple_kfree_callback(struct rcu_head *p)
	{
		struct foo *q = container_of(p, struct foo, rcu);

		kfree(q);
	}

with the following:

	kfree_rcu(&p->rcu, rcu);

Note that the "rcu" is the name of a field in the structure being
freed.  The reason for using this rather than passing in a pointer
to the base structure is that the above approach allows better type
checking.

This commit is based on earlier work by Lai Jiangshan and Manfred Spraul:

Lai's V1 patch: http://lkml.org/lkml/2008/9/18/1
Manfred's patch: http://lkml.org/lkml/2009/1/2/115



Signed-off-by: default avatarLai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: default avatarManfred Spraul <manfred@colorfullife.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarDavid Howells <dhowells@redhat.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent 6cc68793

include/linux/rcupdate.h

+56 −0
Original line number Diff line number Diff line
@@ -809,4 +809,60 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) 
}

#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */



static __always_inline bool __is_kfree_rcu_offset(unsigned long offset)

{

	return offset < 4096;

}



static __always_inline

void __kfree_rcu(struct rcu_head *head, unsigned long offset)

{

	typedef void (*rcu_callback)(struct rcu_head *);



	BUILD_BUG_ON(!__builtin_constant_p(offset));



	/* See the kfree_rcu() header comment. */

	BUILD_BUG_ON(!__is_kfree_rcu_offset(offset));



	call_rcu(head, (rcu_callback)offset);

}



extern void kfree(const void *);



static inline void __rcu_reclaim(struct rcu_head *head)

{

	unsigned long offset = (unsigned long)head->func;



	if (__is_kfree_rcu_offset(offset))

		kfree((void *)head - offset);

	else

		head->func(head);

}



/**

 * kfree_rcu() - kfree an object after a grace period.

 * @ptr:	pointer to kfree

 * @rcu_head:	the name of the struct rcu_head within the type of @ptr.

 *

 * Many rcu callbacks functions just call kfree() on the base structure.

 * These functions are trivial, but their size adds up, and furthermore

 * when they are used in a kernel module, that module must invoke the

 * high-latency rcu_barrier() function at module-unload time.

 *

 * The kfree_rcu() function handles this issue.  Rather than encoding a

 * function address in the embedded rcu_head structure, kfree_rcu() instead

 * encodes the offset of the rcu_head structure within the base structure.

 * Because the functions are not allowed in the low-order 4096 bytes of

 * kernel virtual memory, offsets up to 4095 bytes can be accommodated.

 * If the offset is larger than 4095 bytes, a compile-time error will

 * be generated in __kfree_rcu().  If this error is triggered, you can

 * either fall back to use of call_rcu() or rearrange the structure to

 * position the rcu_head structure into the first 4096 bytes.

 *

 * Note that the allowable offset might decrease in the future, for example,

 * to allow something like kmem_cache_free_rcu().

 */

#define kfree_rcu(ptr, rcu_head)					\

	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))



#endif /* __LINUX_RCUPDATE_H */

kernel/rcutiny.c

+1 −1
Original line number Diff line number Diff line
@@ -167,7 +167,7 @@ static void rcu_process_callbacks(struct rcu_ctrlblk *rcp) 
		prefetch(next);

		debug_rcu_head_unqueue(list);

		local_bh_disable();

		list->func(list);

		__rcu_reclaim(list);

		local_bh_enable();

		list = next;

		RCU_TRACE(cb_count++);

kernel/rcutree.c

+1 −1
Original line number Diff line number Diff line
@@ -1206,7 +1206,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) 
		next = list->next;

		prefetch(next);

		debug_rcu_head_unqueue(list);

		list->func(list);

		__rcu_reclaim(list);

		list = next;

		if (++count >= rdp->blimit)

			break;

CRA Git | Maintained and supported by SUSTech CRA and CCSE