Commit d92a8cfc authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

locking/lockdep: Rework FS_RECLAIM annotation 



A while ago someone, and I cannot find the email just now, asked if we
could not implement the RECLAIM_FS inversion stuff with a 'fake' lock
like we use for other things like workqueues etc. I think this should
be possible which allows reducing the 'irq' states and will reduce the
amount of __bfs() lookups we do.

Removing the 1 IRQ state results in 4 less __bfs() walks per
dependency, improving lockdep performance. And by moving this
annotation out of the lockdep code it becomes easier for the mm people
to extend.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Byungchul Park <byungchul.park@lge.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Nikolay Borisov <nborisov@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: akpm@linux-foundation.org
Cc: boqun.feng@gmail.com
Cc: iamjoonsoo.kim@lge.com
Cc: kernel-team@lge.com
Cc: kirill@shutemov.name
Cc: npiggin@gmail.com
Cc: walken@google.com
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent a9668cd6

drivers/gpu/drm/i915/i915_debugfs.c

+3 −2
Original line number Original line Diff line number Diff line
@@ -28,6 +28,7 @@ 




#include <linux/debugfs.h>

#include <linux/debugfs.h>

#include <linux/sort.h>

#include <linux/sort.h>

#include <linux/sched/mm.h>

#include "intel_drv.h"

#include "intel_drv.h"





static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)

static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
@@ -4331,7 +4332,7 @@ i915_drop_caches_set(void *data, u64 val) 
		mutex_unlock(&dev->struct_mutex);

		mutex_unlock(&dev->struct_mutex);

	}

	}





	lockdep_set_current_reclaim_state(GFP_KERNEL);

	fs_reclaim_acquire(GFP_KERNEL);

	if (val & DROP_BOUND)

	if (val & DROP_BOUND)

		i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND);

		i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND);
@@ -4340,7 +4341,7 @@ i915_drop_caches_set(void *data, u64 val) 




	if (val & DROP_SHRINK_ALL)

	if (val & DROP_SHRINK_ALL)

		i915_gem_shrink_all(dev_priv);

		i915_gem_shrink_all(dev_priv);

	lockdep_clear_current_reclaim_state();

	fs_reclaim_release(GFP_KERNEL);





	if (val & DROP_FREED) {

	if (val & DROP_FREED) {

		synchronize_rcu();

		synchronize_rcu();

include/linux/lockdep.h

+2 −9
Original line number Original line Diff line number Diff line
@@ -29,7 +29,7 @@ extern int lock_stat; 
 * We'd rather not expose kernel/lockdep_states.h this wide, but we do need

 * We'd rather not expose kernel/lockdep_states.h this wide, but we do need

 * the total number of states... :-(

 * the total number of states... :-(

 */

 */

#define XXX_LOCK_USAGE_STATES		(1+3*4)

#define XXX_LOCK_USAGE_STATES		(1+2*4)





/*

/*

 * NR_LOCKDEP_CACHING_CLASSES ... Number of classes

 * NR_LOCKDEP_CACHING_CLASSES ... Number of classes
@@ -363,10 +363,6 @@ static inline void lock_set_subclass(struct lockdep_map *lock, 




extern void lock_downgrade(struct lockdep_map *lock, unsigned long ip);

extern void lock_downgrade(struct lockdep_map *lock, unsigned long ip);





extern void lockdep_set_current_reclaim_state(gfp_t gfp_mask);

extern void lockdep_clear_current_reclaim_state(void);

extern void lockdep_trace_alloc(gfp_t mask);



struct pin_cookie { unsigned int val; };

struct pin_cookie { unsigned int val; };





#define NIL_COOKIE (struct pin_cookie){ .val = 0U, }

#define NIL_COOKIE (struct pin_cookie){ .val = 0U, }
@@ -375,7 +371,7 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock); 
extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);

extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);

extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);

extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);





# define INIT_LOCKDEP				.lockdep_recursion = 0, .lockdep_reclaim_gfp = 0,

# define INIT_LOCKDEP				.lockdep_recursion = 0,





#define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)

#define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)
@@ -416,9 +412,6 @@ static inline void lockdep_on(void) 
# define lock_downgrade(l, i)			do { } while (0)

# define lock_downgrade(l, i)			do { } while (0)

# define lock_set_class(l, n, k, s, i)		do { } while (0)

# define lock_set_class(l, n, k, s, i)		do { } while (0)

# define lock_set_subclass(l, s, i)		do { } while (0)

# define lock_set_subclass(l, s, i)		do { } while (0)

# define lockdep_set_current_reclaim_state(g)	do { } while (0)

# define lockdep_clear_current_reclaim_state()	do { } while (0)

# define lockdep_trace_alloc(g)			do { } while (0)

# define lockdep_info()				do { } while (0)

# define lockdep_info()				do { } while (0)

# define lockdep_init_map(lock, name, key, sub) \

# define lockdep_init_map(lock, name, key, sub) \

		do { (void)(name); (void)(key); } while (0)

		do { (void)(name); (void)(key); } while (0)

include/linux/sched.h

+0 −1
Original line number Original line Diff line number Diff line
@@ -846,7 +846,6 @@ struct task_struct { 
	int				lockdep_depth;

	int				lockdep_depth;

	unsigned int			lockdep_recursion;

	unsigned int			lockdep_recursion;

	struct held_lock		held_locks[MAX_LOCK_DEPTH];

	struct held_lock		held_locks[MAX_LOCK_DEPTH];

	gfp_t				lockdep_reclaim_gfp;

#endif

#endif





#ifdef CONFIG_UBSAN

#ifdef CONFIG_UBSAN

include/linux/sched/mm.h

+8 −0
Original line number Original line Diff line number Diff line
@@ -167,6 +167,14 @@ static inline gfp_t current_gfp_context(gfp_t flags) 
	return flags;

	return flags;

}

}





#ifdef CONFIG_LOCKDEP

extern void fs_reclaim_acquire(gfp_t gfp_mask);

extern void fs_reclaim_release(gfp_t gfp_mask);

#else

static inline void fs_reclaim_acquire(gfp_t gfp_mask) { }

static inline void fs_reclaim_release(gfp_t gfp_mask) { }

#endif



static inline unsigned int memalloc_noio_save(void)

static inline unsigned int memalloc_noio_save(void)

{

{

	unsigned int flags = current->flags & PF_MEMALLOC_NOIO;

	unsigned int flags = current->flags & PF_MEMALLOC_NOIO;

kernel/locking/lockdep.c

+1 −94
Original line number Original line Diff line number Diff line
@@ -344,14 +344,12 @@ EXPORT_SYMBOL(lockdep_on); 
#if VERBOSE

#if VERBOSE

# define HARDIRQ_VERBOSE	1

# define HARDIRQ_VERBOSE	1

# define SOFTIRQ_VERBOSE	1

# define SOFTIRQ_VERBOSE	1

# define RECLAIM_VERBOSE	1

#else

#else

# define HARDIRQ_VERBOSE	0

# define HARDIRQ_VERBOSE	0

# define SOFTIRQ_VERBOSE	0

# define SOFTIRQ_VERBOSE	0

# define RECLAIM_VERBOSE	0

#endif

#endif





#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE || RECLAIM_VERBOSE

#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE

/*

/*

 * Quick filtering for interesting events:

 * Quick filtering for interesting events:

 */

 */
@@ -2567,14 +2565,6 @@ static int SOFTIRQ_verbose(struct lock_class *class) 
	return 0;

	return 0;

}

}





static int RECLAIM_FS_verbose(struct lock_class *class)

{

#if RECLAIM_VERBOSE

	return class_filter(class);

#endif

	return 0;

}



#define STRICT_READ_CHECKS	1

#define STRICT_READ_CHECKS	1





static int (*state_verbose_f[])(struct lock_class *class) = {

static int (*state_verbose_f[])(struct lock_class *class) = {
@@ -2870,57 +2860,6 @@ void trace_softirqs_off(unsigned long ip) 
		debug_atomic_inc(redundant_softirqs_off);

		debug_atomic_inc(redundant_softirqs_off);

}

}





static void __lockdep_trace_alloc(gfp_t gfp_mask, unsigned long flags)

{

	struct task_struct *curr = current;



	if (unlikely(!debug_locks))

		return;



	gfp_mask = current_gfp_context(gfp_mask);



	/* no reclaim without waiting on it */

	if (!(gfp_mask & __GFP_DIRECT_RECLAIM))

		return;



	/* this guy won't enter reclaim */

	if ((curr->flags & PF_MEMALLOC) && !(gfp_mask & __GFP_NOMEMALLOC))

		return;



	/* We're only interested __GFP_FS allocations for now */

	if (!(gfp_mask & __GFP_FS) || (curr->flags & PF_MEMALLOC_NOFS))

		return;



	/*

	 * Oi! Can't be having __GFP_FS allocations with IRQs disabled.

	 */

	if (DEBUG_LOCKS_WARN_ON(irqs_disabled_flags(flags)))

		return;



	/* Disable lockdep if explicitly requested */

	if (gfp_mask & __GFP_NOLOCKDEP)

		return;



	mark_held_locks(curr, RECLAIM_FS);

}



static void check_flags(unsigned long flags);



void lockdep_trace_alloc(gfp_t gfp_mask)

{

	unsigned long flags;



	if (unlikely(current->lockdep_recursion))

		return;



	raw_local_irq_save(flags);

	check_flags(flags);

	current->lockdep_recursion = 1;

	__lockdep_trace_alloc(gfp_mask, flags);

	current->lockdep_recursion = 0;

	raw_local_irq_restore(flags);

}



static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)

static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)

{

{

	/*

	/*
@@ -2966,22 +2905,6 @@ static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock) 
		}

		}

	}

	}





	/*

	 * We reuse the irq context infrastructure more broadly as a general

	 * context checking code. This tests GFP_FS recursion (a lock taken

	 * during reclaim for a GFP_FS allocation is held over a GFP_FS

	 * allocation).

	 */

	if (!hlock->trylock && (curr->lockdep_reclaim_gfp & __GFP_FS)) {

		if (hlock->read) {

			if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS_READ))

					return 0;

		} else {

			if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS))

					return 0;

		}

	}



	return 1;

	return 1;

}

}
@@ -3040,10 +2963,6 @@ static inline int separate_irq_context(struct task_struct *curr, 
	return 0;

	return 0;

}

}





void lockdep_trace_alloc(gfp_t gfp_mask)

{

}



#endif /* defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING) */

#endif /* defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING) */





/*

/*
@@ -3952,18 +3871,6 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie) 
}

}

EXPORT_SYMBOL_GPL(lock_unpin_lock);

EXPORT_SYMBOL_GPL(lock_unpin_lock);





void lockdep_set_current_reclaim_state(gfp_t gfp_mask)

{

	current->lockdep_reclaim_gfp = current_gfp_context(gfp_mask);

}

EXPORT_SYMBOL_GPL(lockdep_set_current_reclaim_state);



void lockdep_clear_current_reclaim_state(void)

{

	current->lockdep_reclaim_gfp = 0;

}

EXPORT_SYMBOL_GPL(lockdep_clear_current_reclaim_state);



#ifdef CONFIG_LOCK_STAT

#ifdef CONFIG_LOCK_STAT

static int

static int

print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock,

print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock,

CRA Git | Maintained and supported by SUSTech CRA and CCSE