Commit 1cde2930 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched/preempt: Add static_key() to preempt_notifiers 



Avoid touching the curr->preempt_notifier cacheline when not needed.

Provides a small improvement on pipe-bench:

  taskset 01 perf stat --repeat 10 -- perf bench sched pipe

before:

 Performance counter stats for 'perf bench sched pipe' (10 runs):

      12385.016204      task-clock (msec)         #    1.001 CPUs utilized            ( +-  0.34% )
         2,000,023      context-switches          #    0.161 M/sec                    ( +-  0.00% )
                 0      cpu-migrations            #    0.000 K/sec
               175      page-faults               #    0.014 K/sec                    ( +-  0.26% )
    41,376,162,250      cycles                    #    3.341 GHz                      ( +-  0.11% )
    17,389,139,321      stalled-cycles-frontend   #   42.03% frontend cycles idle     ( +-  0.25% )
   <not supported>      stalled-cycles-backend
    68,788,588,003      instructions              #    1.66  insns per cycle
                                                  #    0.25  stalled cycles per insn  ( +-  0.02% )
    13,449,387,620      branches                  # 1085.940 M/sec                    ( +-  0.02% )
        20,880,690      branch-misses             #    0.16% of all branches          ( +-  0.98% )

      12.372646094 seconds time elapsed                                          ( +-  0.34% )

after:

 Performance counter stats for 'perf bench sched pipe' (10 runs):

      12180.936528      task-clock (msec)         #    1.001 CPUs utilized            ( +-  0.33% )
         2,000,077      context-switches          #    0.164 M/sec                    ( +-  0.00% )
                 0      cpu-migrations            #    0.000 K/sec
               174      page-faults               #    0.014 K/sec                    ( +-  0.27% )
    40,691,545,577      cycles                    #    3.341 GHz                      ( +-  0.06% )
    16,446,333,371      stalled-cycles-frontend   #   40.42% frontend cycles idle     ( +-  0.18% )
   <not supported>      stalled-cycles-backend
    68,570,100,387      instructions              #    1.69  insns per cycle
                                                  #    0.24  stalled cycles per insn  ( +-  0.01% )
    13,389,740,014      branches                  # 1099.237 M/sec                    ( +-  0.01% )
        20,175,440      branch-misses             #    0.15% of all branches          ( +-  0.52% )

      12.169253010 seconds time elapsed                                          ( +-  0.33% )

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent d84525a8

kernel/sched/core.c

+23 −5
Original line number Diff line number Diff line
@@ -2151,12 +2151,15 @@ void wake_up_new_task(struct task_struct *p) 


#ifdef CONFIG_PREEMPT_NOTIFIERS



static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE;



/**

 * preempt_notifier_register - tell me when current is being preempted & rescheduled

 * @notifier: notifier struct to register

 */

void preempt_notifier_register(struct preempt_notifier *notifier)

{

	static_key_slow_inc(&preempt_notifier_key);

	hlist_add_head(&notifier->link, &current->preempt_notifiers);

}

EXPORT_SYMBOL_GPL(preempt_notifier_register);
@@ -2170,10 +2173,11 @@ EXPORT_SYMBOL_GPL(preempt_notifier_register); 
void preempt_notifier_unregister(struct preempt_notifier *notifier)

{

	hlist_del(&notifier->link);

	static_key_slow_dec(&preempt_notifier_key);

}

EXPORT_SYMBOL_GPL(preempt_notifier_unregister);



static void fire_sched_in_preempt_notifiers(struct task_struct *curr)

static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)

{

	struct preempt_notifier *notifier;
@@ -2181,8 +2185,14 @@ static void fire_sched_in_preempt_notifiers(struct task_struct *curr) 
		notifier->ops->sched_in(notifier, raw_smp_processor_id());

}



static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)

{

	if (static_key_false(&preempt_notifier_key))

		__fire_sched_in_preempt_notifiers(curr);

}



static void

fire_sched_out_preempt_notifiers(struct task_struct *curr,

__fire_sched_out_preempt_notifiers(struct task_struct *curr,

				   struct task_struct *next)

{

	struct preempt_notifier *notifier;
@@ -2191,13 +2201,21 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr, 
		notifier->ops->sched_out(notifier, next);

}



static __always_inline void

fire_sched_out_preempt_notifiers(struct task_struct *curr,

				 struct task_struct *next)

{

	if (static_key_false(&preempt_notifier_key))

		__fire_sched_out_preempt_notifiers(curr, next);

}



#else /* !CONFIG_PREEMPT_NOTIFIERS */



static void fire_sched_in_preempt_notifiers(struct task_struct *curr)

static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)

{

}



static void

static inline void

fire_sched_out_preempt_notifiers(struct task_struct *curr,

				 struct task_struct *next)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE