Commit f82b217e authored by Dave Jones's avatar Dave Jones Committed by Ingo Molnar
Browse files

lockdep: shrink held_lock structure 



struct held_lock {
        u64                        prev_chain_key;       /*     0     8 */
        struct lock_class *        class;                /*     8     8 */
        long unsigned int          acquire_ip;           /*    16     8 */
        struct lockdep_map *       instance;             /*    24     8 */
        int                        irq_context;          /*    32     4 */
        int                        trylock;              /*    36     4 */
        int                        read;                 /*    40     4 */
        int                        check;                /*    44     4 */
        int                        hardirqs_off;         /*    48     4 */

        /* size: 56, cachelines: 1 */
        /* padding: 4 */
        /* last cacheline: 56 bytes */
};

struct held_lock {
        u64                        prev_chain_key;       /*     0     8 */
        long unsigned int          acquire_ip;           /*     8     8 */
        struct lockdep_map *       instance;             /*    16     8 */
        unsigned int               class_idx:11;         /*    24:21  4 */
        unsigned int               irq_context:2;        /*    24:19  4 */
        unsigned int               trylock:1;            /*    24:18  4 */
        unsigned int               read:2;               /*    24:16  4 */
        unsigned int               check:2;              /*    24:14  4 */
        unsigned int               hardirqs_off:1;       /*    24:13  4 */

        /* size: 32, cachelines: 1 */
        /* padding: 4 */
        /* bit_padding: 13 bits */
        /* last cacheline: 32 bytes */
};

[mingo@elte.hu: shrunk hlock->class too]
[peterz@infradead.org: fixup bit sizes]
Signed-off-by: default avatarDave Jones <davej@redhat.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
parent 1b12bbc7

include/linux/lockdep.h

+9 −7
Original line number Diff line number Diff line
@@ -190,6 +190,9 @@ struct lock_chain { 
	u64				chain_key;

};



#define MAX_LOCKDEP_KEYS_BITS		11

#define MAX_LOCKDEP_KEYS		(1UL << MAX_LOCKDEP_KEYS_BITS)



struct held_lock {

	/*

	 * One-way hash of the dependency chain up to this point. We
@@ -206,14 +209,13 @@ struct held_lock { 
	 * with zero), here we store the previous hash value:

	 */

	u64				prev_chain_key;

	struct lock_class		*class;

	unsigned long			acquire_ip;

	struct lockdep_map		*instance;



#ifdef CONFIG_LOCK_STAT

	u64 				waittime_stamp;

	u64				holdtime_stamp;

#endif

	unsigned int			class_idx:MAX_LOCKDEP_KEYS_BITS;

	/*

	 * The lock-stack is unified in that the lock chains of interrupt

	 * contexts nest ontop of process context chains, but we 'separate'
@@ -227,11 +229,11 @@ struct held_lock { 
	 * The following field is used to detect when we cross into an

	 * interrupt context:

	 */

	int				irq_context;

	int				trylock;

	int				read;

	int				check;

	int				hardirqs_off;

	unsigned int irq_context:2; /* bit 0 - soft, bit 1 - hard */

	unsigned int trylock:1;

	unsigned int read:2;        /* see lock_acquire() comment */

	unsigned int check:2;       /* see lock_acquire() comment */

	unsigned int hardirqs_off:1;

};



/*

kernel/lockdep.c

+65 −48
Original line number Diff line number Diff line
@@ -124,6 +124,15 @@ static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES]; 
unsigned long nr_lock_classes;

static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];



static inline struct lock_class *hlock_class(struct held_lock *hlock)

{

	if (!hlock->class_idx) {

		DEBUG_LOCKS_WARN_ON(1);

		return NULL;

	}

	return lock_classes + hlock->class_idx - 1;

}



#ifdef CONFIG_LOCK_STAT

static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats);
@@ -222,7 +231,7 @@ static void lock_release_holdtime(struct held_lock *hlock) 


	holdtime = sched_clock() - hlock->holdtime_stamp;



	stats = get_lock_stats(hlock->class);

	stats = get_lock_stats(hlock_class(hlock));

	if (hlock->read)

		lock_time_inc(&stats->read_holdtime, holdtime);

	else
@@ -518,7 +527,7 @@ static void print_lockdep_cache(struct lockdep_map *lock) 


static void print_lock(struct held_lock *hlock)

{

	print_lock_name(hlock->class);

	print_lock_name(hlock_class(hlock));

	printk(", at: ");

	print_ip_sym(hlock->acquire_ip);

}
@@ -948,7 +957,7 @@ static noinline int print_circular_bug_tail(void) 
	if (debug_locks_silent)

		return 0;



	this.class = check_source->class;

	this.class = hlock_class(check_source);

	if (!save_trace(&this.trace))

		return 0;
@@ -1057,7 +1066,7 @@ check_noncircular(struct lock_class *source, unsigned int depth) 
	 * Check this lock's dependency list:

	 */

	list_for_each_entry(entry, &source->locks_after, entry) {

		if (entry->class == check_target->class)

		if (entry->class == hlock_class(check_target))

			return print_circular_bug_header(entry, depth+1);

		debug_atomic_inc(&nr_cyclic_checks);

		if (!check_noncircular(entry->class, depth+1))
@@ -1150,6 +1159,11 @@ find_usage_backwards(struct lock_class *source, unsigned int depth) 
		return 2;

	}



	if (!source && debug_locks_off_graph_unlock()) {

		WARN_ON(1);

		return 0;

	}



	/*

	 * Check this lock's dependency list:

	 */
@@ -1189,9 +1203,9 @@ print_bad_irq_dependency(struct task_struct *curr, 
	printk("\nand this task is already holding:\n");

	print_lock(prev);

	printk("which would create a new lock dependency:\n");

	print_lock_name(prev->class);

	print_lock_name(hlock_class(prev));

	printk(" ->");

	print_lock_name(next->class);

	print_lock_name(hlock_class(next));

	printk("\n");



	printk("\nbut this new dependency connects a %s-irq-safe lock:\n",
@@ -1232,12 +1246,12 @@ check_usage(struct task_struct *curr, struct held_lock *prev, 


	find_usage_bit = bit_backwards;

	/* fills in <backwards_match> */

	ret = find_usage_backwards(prev->class, 0);

	ret = find_usage_backwards(hlock_class(prev), 0);

	if (!ret || ret == 1)

		return ret;



	find_usage_bit = bit_forwards;

	ret = find_usage_forwards(next->class, 0);

	ret = find_usage_forwards(hlock_class(next), 0);

	if (!ret || ret == 1)

		return ret;

	/* ret == 2 */
@@ -1362,7 +1376,7 @@ check_deadlock(struct task_struct *curr, struct held_lock *next, 


	for (i = 0; i < curr->lockdep_depth; i++) {

		prev = curr->held_locks + i;

		if (prev->class != next->class)

		if (hlock_class(prev) != hlock_class(next))

			continue;

		/*

		 * Allow read-after-read recursion of the same
@@ -1415,7 +1429,7 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev, 
	 */

	check_source = next;

	check_target = prev;

	if (!(check_noncircular(next->class, 0)))

	if (!(check_noncircular(hlock_class(next), 0)))

		return print_circular_bug_tail();



	if (!check_prev_add_irq(curr, prev, next))
@@ -1439,8 +1453,8 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev, 
	 *  chains - the second one will be new, but L1 already has

	 *  L2 added to its dependency list, due to the first chain.)

	 */

	list_for_each_entry(entry, &prev->class->locks_after, entry) {

		if (entry->class == next->class) {

	list_for_each_entry(entry, &hlock_class(prev)->locks_after, entry) {

		if (entry->class == hlock_class(next)) {

			if (distance == 1)

				entry->distance = 1;

			return 2;
@@ -1451,26 +1465,28 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev, 
	 * Ok, all validations passed, add the new lock

	 * to the previous lock's dependency list:

	 */

	ret = add_lock_to_list(prev->class, next->class,

			       &prev->class->locks_after, next->acquire_ip, distance);

	ret = add_lock_to_list(hlock_class(prev), hlock_class(next),

			       &hlock_class(prev)->locks_after,

			       next->acquire_ip, distance);



	if (!ret)

		return 0;



	ret = add_lock_to_list(next->class, prev->class,

			       &next->class->locks_before, next->acquire_ip, distance);

	ret = add_lock_to_list(hlock_class(next), hlock_class(prev),

			       &hlock_class(next)->locks_before,

			       next->acquire_ip, distance);

	if (!ret)

		return 0;



	/*

	 * Debugging printouts:

	 */

	if (verbose(prev->class) || verbose(next->class)) {

	if (verbose(hlock_class(prev)) || verbose(hlock_class(next))) {

		graph_unlock();

		printk("\n new dependency: ");

		print_lock_name(prev->class);

		print_lock_name(hlock_class(prev));

		printk(" => ");

		print_lock_name(next->class);

		print_lock_name(hlock_class(next));

		printk("\n");

		dump_stack();

		return graph_lock();
@@ -1567,7 +1583,7 @@ static inline int lookup_chain_cache(struct task_struct *curr, 
				     struct held_lock *hlock,

				     u64 chain_key)

{

	struct lock_class *class = hlock->class;

	struct lock_class *class = hlock_class(hlock);

	struct list_head *hash_head = chainhashentry(chain_key);

	struct lock_chain *chain;

	struct held_lock *hlock_curr, *hlock_next;
@@ -1640,7 +1656,7 @@ cache_hit: 
	if (likely(cn + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {

		chain->base = cn;

		for (j = 0; j < chain->depth - 1; j++, i++) {

			int lock_id = curr->held_locks[i].class - lock_classes;

			int lock_id = curr->held_locks[i].class_idx - 1;

			chain_hlocks[chain->base + j] = lock_id;

		}

		chain_hlocks[chain->base + j] = class - lock_classes;
@@ -1736,7 +1752,7 @@ static void check_chain_key(struct task_struct *curr) 
			WARN_ON(1);

			return;

		}

		id = hlock->class - lock_classes;

		id = hlock->class_idx - 1;

		if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS))

			return;
@@ -1781,7 +1797,7 @@ print_usage_bug(struct task_struct *curr, struct held_lock *this, 
	print_lock(this);



	printk("{%s} state was registered at:\n", usage_str[prev_bit]);

	print_stack_trace(this->class->usage_traces + prev_bit, 1);

	print_stack_trace(hlock_class(this)->usage_traces + prev_bit, 1);



	print_irqtrace_events(curr);

	printk("\nother info that might help us debug this:\n");
@@ -1800,7 +1816,7 @@ static inline int 
valid_state(struct task_struct *curr, struct held_lock *this,

	    enum lock_usage_bit new_bit, enum lock_usage_bit bad_bit)

{

	if (unlikely(this->class->usage_mask & (1 << bad_bit)))

	if (unlikely(hlock_class(this)->usage_mask & (1 << bad_bit)))

		return print_usage_bug(curr, this, bad_bit, new_bit);

	return 1;

}
@@ -1839,7 +1855,7 @@ print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other, 
	lockdep_print_held_locks(curr);



	printk("\nthe first lock's dependencies:\n");

	print_lock_dependencies(this->class, 0);

	print_lock_dependencies(hlock_class(this), 0);



	printk("\nthe second lock's dependencies:\n");

	print_lock_dependencies(other, 0);
@@ -1862,7 +1878,7 @@ check_usage_forwards(struct task_struct *curr, struct held_lock *this, 


	find_usage_bit = bit;

	/* fills in <forwards_match> */

	ret = find_usage_forwards(this->class, 0);

	ret = find_usage_forwards(hlock_class(this), 0);

	if (!ret || ret == 1)

		return ret;
@@ -1881,7 +1897,7 @@ check_usage_backwards(struct task_struct *curr, struct held_lock *this, 


	find_usage_bit = bit;

	/* fills in <backwards_match> */

	ret = find_usage_backwards(this->class, 0);

	ret = find_usage_backwards(hlock_class(this), 0);

	if (!ret || ret == 1)

		return ret;
@@ -1947,7 +1963,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
				LOCK_ENABLED_HARDIRQS_READ, "hard-read"))

			return 0;

#endif

		if (hardirq_verbose(this->class))

		if (hardirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	case LOCK_USED_IN_SOFTIRQ:
@@ -1972,7 +1988,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
				LOCK_ENABLED_SOFTIRQS_READ, "soft-read"))

			return 0;

#endif

		if (softirq_verbose(this->class))

		if (softirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	case LOCK_USED_IN_HARDIRQ_READ:
@@ -1985,7 +2001,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
		if (!check_usage_forwards(curr, this,

					  LOCK_ENABLED_HARDIRQS, "hard"))

			return 0;

		if (hardirq_verbose(this->class))

		if (hardirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	case LOCK_USED_IN_SOFTIRQ_READ:
@@ -1998,7 +2014,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
		if (!check_usage_forwards(curr, this,

					  LOCK_ENABLED_SOFTIRQS, "soft"))

			return 0;

		if (softirq_verbose(this->class))

		if (softirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	case LOCK_ENABLED_HARDIRQS:
@@ -2024,7 +2040,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
				   LOCK_USED_IN_HARDIRQ_READ, "hard-read"))

			return 0;

#endif

		if (hardirq_verbose(this->class))

		if (hardirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	case LOCK_ENABLED_SOFTIRQS:
@@ -2050,7 +2066,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
				   LOCK_USED_IN_SOFTIRQ_READ, "soft-read"))

			return 0;

#endif

		if (softirq_verbose(this->class))

		if (softirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	case LOCK_ENABLED_HARDIRQS_READ:
@@ -2065,7 +2081,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
					   LOCK_USED_IN_HARDIRQ, "hard"))

			return 0;

#endif

		if (hardirq_verbose(this->class))

		if (hardirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	case LOCK_ENABLED_SOFTIRQS_READ:
@@ -2080,7 +2096,7 @@ static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 
					   LOCK_USED_IN_SOFTIRQ, "soft"))

			return 0;

#endif

		if (softirq_verbose(this->class))

		if (softirq_verbose(hlock_class(this)))

			ret = 2;

		break;

	default:
@@ -2396,7 +2412,7 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this, 
	 * If already set then do not dirty the cacheline,

	 * nor do any checks:

	 */

	if (likely(this->class->usage_mask & new_mask))

	if (likely(hlock_class(this)->usage_mask & new_mask))

		return 1;



	if (!graph_lock())
@@ -2404,14 +2420,14 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this, 
	/*

	 * Make sure we didnt race:

	 */

	if (unlikely(this->class->usage_mask & new_mask)) {

	if (unlikely(hlock_class(this)->usage_mask & new_mask)) {

		graph_unlock();

		return 1;

	}



	this->class->usage_mask |= new_mask;

	hlock_class(this)->usage_mask |= new_mask;



	if (!save_trace(this->class->usage_traces + new_bit))

	if (!save_trace(hlock_class(this)->usage_traces + new_bit))

		return 0;



	switch (new_bit) {
@@ -2545,8 +2561,9 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, 
		return 0;



	hlock = curr->held_locks + depth;



	hlock->class = class;

	if (DEBUG_LOCKS_WARN_ON(!class))

		return 0;

	hlock->class_idx = class - lock_classes + 1;

	hlock->acquire_ip = ip;

	hlock->instance = lock;

	hlock->trylock = trylock;
@@ -2690,7 +2707,7 @@ __lock_set_subclass(struct lockdep_map *lock, 


found_it:

	class = register_lock_class(lock, subclass, 0);

	hlock->class = class;

	hlock->class_idx = class - lock_classes + 1;



	curr->lockdep_depth = i;

	curr->curr_chain_key = hlock->prev_chain_key;
@@ -2698,7 +2715,7 @@ found_it: 
	for (; i < depth; i++) {

		hlock = curr->held_locks + i;

		if (!__lock_acquire(hlock->instance,

			hlock->class->subclass, hlock->trylock,

			hlock_class(hlock)->subclass, hlock->trylock,

				hlock->read, hlock->check, hlock->hardirqs_off,

				hlock->acquire_ip))

			return 0;
@@ -2759,7 +2776,7 @@ found_it: 
	for (i++; i < depth; i++) {

		hlock = curr->held_locks + i;

		if (!__lock_acquire(hlock->instance,

			hlock->class->subclass, hlock->trylock,

			hlock_class(hlock)->subclass, hlock->trylock,

				hlock->read, hlock->check, hlock->hardirqs_off,

				hlock->acquire_ip))

			return 0;
@@ -2804,7 +2821,7 @@ static int lock_release_nested(struct task_struct *curr, 


#ifdef CONFIG_DEBUG_LOCKDEP

	hlock->prev_chain_key = 0;

	hlock->class = NULL;

	hlock->class_idx = 0;

	hlock->acquire_ip = 0;

	hlock->irq_context = 0;

#endif
@@ -3000,9 +3017,9 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip) 
found_it:

	hlock->waittime_stamp = sched_clock();



	point = lock_contention_point(hlock->class, ip);

	point = lock_contention_point(hlock_class(hlock), ip);



	stats = get_lock_stats(hlock->class);

	stats = get_lock_stats(hlock_class(hlock));

	if (point < ARRAY_SIZE(stats->contention_point))

		stats->contention_point[i]++;

	if (lock->cpu != smp_processor_id())
@@ -3048,7 +3065,7 @@ found_it: 
		hlock->holdtime_stamp = now;

	}



	stats = get_lock_stats(hlock->class);

	stats = get_lock_stats(hlock_class(hlock));

	if (waittime) {

		if (hlock->read)

			lock_time_inc(&stats->read_waittime, waittime);

kernel/lockdep_internals.h

+0 −3
Original line number Diff line number Diff line
@@ -17,9 +17,6 @@ 
 */

#define MAX_LOCKDEP_ENTRIES	8192UL



#define MAX_LOCKDEP_KEYS_BITS	11

#define MAX_LOCKDEP_KEYS	(1UL << MAX_LOCKDEP_KEYS_BITS)



#define MAX_LOCKDEP_CHAINS_BITS	14

#define MAX_LOCKDEP_CHAINS	(1UL << MAX_LOCKDEP_CHAINS_BITS)

CRA Git | Maintained and supported by SUSTech CRA and CCSE