Commit 8ca2b56c authored by Waiman Long's avatar Waiman Long Committed by Ingo Molnar
Browse files

locking/lockdep: Make class->ops a percpu counter and move it under CONFIG_DEBUG_LOCKDEP=y 



A sizable portion of the CPU cycles spent on the __lock_acquire() is used
up by the atomic increment of the class->ops stat counter. By taking it out
from the lock_class structure and changing it to a per-cpu per-lock-class
counter, we can reduce the amount of cacheline contention on the class
structure when multiple CPUs are trying to acquire locks of the same
class simultaneously.

To limit the increase in memory consumption because of the percpu nature
of that counter, it is now put back under the CONFIG_DEBUG_LOCKDEP
config option. So the memory consumption increase will only occur if
CONFIG_DEBUG_LOCKDEP is defined. The lock_class structure, however,
is reduced in size by 16 bytes on 64-bit archs after ops removal and
a minor restructuring of the fields.

This patch also fixes a bug in the increment code as the counter is of
the 'unsigned long' type, but atomic_inc() was used to increment it.

Signed-off-by: default avatarWaiman Long <longman@redhat.com>
Acked-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will.deacon@arm.com>
Link: http://lkml.kernel.org/r/d66681f3-8781-9793-1dcf-2436a284550b@redhat.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent ce52a18d

include/linux/lockdep.h

+1 −6
Original line number Original line Diff line number Diff line
@@ -99,13 +99,8 @@ struct lock_class { 
	 */

	 */

	unsigned int			version;

	unsigned int			version;





	/*

	 * Statistics counter:

	 */

	unsigned long			ops;



	const char			*name;

	int				name_version;

	int				name_version;

	const char			*name;





#ifdef CONFIG_LOCK_STAT

#ifdef CONFIG_LOCK_STAT

	unsigned long			contention_point[LOCKSTAT_POINTS];

	unsigned long			contention_point[LOCKSTAT_POINTS];

kernel/locking/lockdep.c

+8 −3
Original line number Original line Diff line number Diff line
@@ -139,7 +139,7 @@ static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES]; 
 * get freed - this significantly simplifies the debugging code.

 * get freed - this significantly simplifies the debugging code.

 */

 */

unsigned long nr_lock_classes;

unsigned long nr_lock_classes;

static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];

struct lock_class lock_classes[MAX_LOCKDEP_KEYS];





static inline struct lock_class *hlock_class(struct held_lock *hlock)

static inline struct lock_class *hlock_class(struct held_lock *hlock)

{

{
@@ -436,6 +436,7 @@ unsigned int max_lockdep_depth; 
 * Various lockdep statistics:

 * Various lockdep statistics:

 */

 */

DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);

DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);

DEFINE_PER_CPU(unsigned long [MAX_LOCKDEP_KEYS], lock_class_ops);

#endif

#endif





/*

/*
@@ -1392,7 +1393,9 @@ static void print_lock_class_header(struct lock_class *class, int depth) 




	printk("%*s->", depth, "");

	printk("%*s->", depth, "");

	print_lock_name(class);

	print_lock_name(class);

	printk(KERN_CONT " ops: %lu", class->ops);

#ifdef CONFIG_DEBUG_LOCKDEP

	printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));

#endif

	printk(KERN_CONT " {\n");

	printk(KERN_CONT " {\n");





	for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {

	for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
@@ -3227,7 +3230,9 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, 
		if (!class)

		if (!class)

			return 0;

			return 0;

	}

	}

	atomic_inc((atomic_t *)&class->ops);



	debug_class_ops_inc(class);



	if (very_verbose(class)) {

	if (very_verbose(class)) {

		printk("\nacquire class [%px] %s", class->key, class->name);

		printk("\nacquire class [%px] %s", class->key, class->name);

		if (class->name_version > 1)

		if (class->name_version > 1)

kernel/locking/lockdep_internals.h

+27 −0
Original line number Original line Diff line number Diff line
@@ -152,9 +152,15 @@ struct lockdep_stats { 
	int	nr_find_usage_forwards_recursions;

	int	nr_find_usage_forwards_recursions;

	int	nr_find_usage_backwards_checks;

	int	nr_find_usage_backwards_checks;

	int	nr_find_usage_backwards_recursions;

	int	nr_find_usage_backwards_recursions;



	/*

	 * Per lock class locking operation stat counts

	 */

	unsigned long lock_class_ops[MAX_LOCKDEP_KEYS];

};

};





DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);

DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);

extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS];





#define __debug_atomic_inc(ptr)					\

#define __debug_atomic_inc(ptr)					\

	this_cpu_inc(lockdep_stats.ptr);

	this_cpu_inc(lockdep_stats.ptr);
@@ -179,9 +185,30 @@ DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats); 
	}								\

	}								\

	__total;							\

	__total;							\

})

})



static inline void debug_class_ops_inc(struct lock_class *class)

{

	int idx;



	idx = class - lock_classes;

	__debug_atomic_inc(lock_class_ops[idx]);

}



static inline unsigned long debug_class_ops_read(struct lock_class *class)

{

	int idx, cpu;

	unsigned long ops = 0;



	idx = class - lock_classes;

	for_each_possible_cpu(cpu)

		ops += per_cpu(lockdep_stats.lock_class_ops[idx], cpu);

	return ops;

}



#else

#else

# define __debug_atomic_inc(ptr)	do { } while (0)

# define __debug_atomic_inc(ptr)	do { } while (0)

# define debug_atomic_inc(ptr)		do { } while (0)

# define debug_atomic_inc(ptr)		do { } while (0)

# define debug_atomic_dec(ptr)		do { } while (0)

# define debug_atomic_dec(ptr)		do { } while (0)

# define debug_atomic_read(ptr)		0

# define debug_atomic_read(ptr)		0

# define debug_class_ops_inc(ptr)	do { } while (0)

#endif
 
#endif

kernel/locking/lockdep_proc.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -68,7 +68,7 @@ static int l_show(struct seq_file *m, void *v) 




	seq_printf(m, "%p", class->key);

	seq_printf(m, "%p", class->key);

#ifdef CONFIG_DEBUG_LOCKDEP

#ifdef CONFIG_DEBUG_LOCKDEP

	seq_printf(m, " OPS:%8ld", class->ops);

	seq_printf(m, " OPS:%8ld", debug_class_ops_read(class));

#endif

#endif

#ifdef CONFIG_PROVE_LOCKING

#ifdef CONFIG_PROVE_LOCKING

	seq_printf(m, " FD:%5ld", lockdep_count_forward_deps(class));

	seq_printf(m, " FD:%5ld", lockdep_count_forward_deps(class));

CRA Git | Maintained and supported by SUSTech CRA and CCSE