Commit 370504cf authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched 

* git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched:
  [PATCH] sched: reduce task_struct size
  [PATCH] sched: reduce debug code
  [PATCH] sched: use schedstat_set() API
  [PATCH] sched: add schedstat_set() API
  [PATCH] sched: move load-calculation functions
  [PATCH] sched: ->task_new cleanup
  [PATCH] sched: uninline inc/dec_nr_running()
  [PATCH] sched: uninline calc_delta_mine()
  [PATCH] sched: calc_delta_mine(): use fixed limit
  [PATCH] sched: tidy up left over smpnice code
  [PATCH] sched: remove cache_hot_time
parents 160d6aaf 94c18227

include/linux/sched.h

+14 −10
Original line number Diff line number Diff line
@@ -734,7 +734,6 @@ struct sched_domain { 
	unsigned long max_interval;	/* Maximum balance interval ms */

	unsigned int busy_factor;	/* less balancing by factor if busy */

	unsigned int imbalance_pct;	/* No balance until over watermark */

	unsigned long long cache_hot_time; /* Task considered cache hot (ns) */

	unsigned int cache_nice_tries;	/* Leave cache hot tasks for # tries */

	unsigned int busy_idx;

	unsigned int idle_idx;
@@ -875,7 +874,7 @@ struct sched_class { 


	void (*set_curr_task) (struct rq *rq);

	void (*task_tick) (struct rq *rq, struct task_struct *p);

	void (*task_new) (struct rq *rq, struct task_struct *p);

	void (*task_new) (struct rq *rq, struct task_struct *p, u64 now);

};



struct load_weight {
@@ -905,23 +904,28 @@ struct sched_entity { 
	struct rb_node		run_node;

	unsigned int		on_rq;



	u64			exec_start;

	u64			sum_exec_runtime;

	u64			wait_start_fair;

	u64			sleep_start_fair;



#ifdef CONFIG_SCHEDSTATS

	u64			wait_start;

	u64			exec_start;

	u64			wait_max;

	s64			sum_wait_runtime;



	u64			sleep_start;

	u64			sleep_start_fair;

	u64			block_start;

	u64			sleep_max;

	s64			sum_sleep_runtime;



	u64			block_start;

	u64			block_max;

	u64			exec_max;

	u64			wait_max;

	u64			last_ran;



	u64			sum_exec_runtime;

	s64			sum_wait_runtime;

	s64			sum_sleep_runtime;

	unsigned long		wait_runtime_overruns;

	unsigned long		wait_runtime_underruns;

#endif



#ifdef CONFIG_FAIR_GROUP_SCHED

	struct sched_entity	*parent;

	/* rq on which this entity is (to be) queued: */

include/linux/topology.h

+0 −1
Original line number Diff line number Diff line
@@ -185,7 +185,6 @@ 
	.max_interval		= 64*num_online_cpus(),	\

	.busy_factor		= 128,			\

	.imbalance_pct		= 133,			\

	.cache_hot_time		= (10*1000000),		\

	.cache_nice_tries	= 1,			\

	.busy_idx		= 3,			\

	.idle_idx		= 3,			\

kernel/sched.c

+95 −98
Original line number Diff line number Diff line
@@ -637,7 +637,7 @@ static u64 div64_likely32(u64 divident, unsigned long divisor) 


#define WMULT_SHIFT	32



static inline unsigned long

static unsigned long

calc_delta_mine(unsigned long delta_exec, unsigned long weight,

		struct load_weight *lw)

{
@@ -657,7 +657,7 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight, 
		tmp = (tmp * lw->inv_weight) >> WMULT_SHIFT;

	}



	return (unsigned long)min(tmp, (u64)sysctl_sched_runtime_limit);

	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);

}



static inline unsigned long
@@ -678,46 +678,6 @@ static void update_load_sub(struct load_weight *lw, unsigned long dec) 
	lw->inv_weight = 0;

}



static void __update_curr_load(struct rq *rq, struct load_stat *ls)

{

	if (rq->curr != rq->idle && ls->load.weight) {

		ls->delta_exec += ls->delta_stat;

		ls->delta_fair += calc_delta_fair(ls->delta_stat, &ls->load);

		ls->delta_stat = 0;

	}

}



/*

 * Update delta_exec, delta_fair fields for rq.

 *

 * delta_fair clock advances at a rate inversely proportional to

 * total load (rq->ls.load.weight) on the runqueue, while

 * delta_exec advances at the same rate as wall-clock (provided

 * cpu is not idle).

 *

 * delta_exec / delta_fair is a measure of the (smoothened) load on this

 * runqueue over any given interval. This (smoothened) load is used

 * during load balance.

 *

 * This function is called /before/ updating rq->ls.load

 * and when switching tasks.

 */

static void update_curr_load(struct rq *rq, u64 now)

{

	struct load_stat *ls = &rq->ls;

	u64 start;



	start = ls->load_update_start;

	ls->load_update_start = now;

	ls->delta_stat += now - start;

	/*

	 * Stagger updates to ls->delta_fair. Very frequent updates

	 * can be expensive.

	 */

	if (ls->delta_stat >= sysctl_sched_stat_granularity)

		__update_curr_load(rq, ls);

}



/*

 * To aid in avoiding the subversion of "niceness" due to uneven distribution

 * of tasks with abnormal "nice" values across CPUs the contribution that
@@ -727,19 +687,6 @@ static void update_curr_load(struct rq *rq, u64 now) 
 * slice expiry etc.

 */



/*

 * Assume: static_prio_timeslice(NICE_TO_PRIO(0)) == DEF_TIMESLICE

 * If static_prio_timeslice() is ever changed to break this assumption then

 * this code will need modification

 */

#define TIME_SLICE_NICE_ZERO DEF_TIMESLICE

#define load_weight(lp) \

	(((lp) * SCHED_LOAD_SCALE) / TIME_SLICE_NICE_ZERO)

#define PRIO_TO_LOAD_WEIGHT(prio) \

	load_weight(static_prio_timeslice(prio))

#define RTPRIO_TO_LOAD_WEIGHT(rp) \

	(PRIO_TO_LOAD_WEIGHT(MAX_RT_PRIO) + load_weight(rp))



#define WEIGHT_IDLEPRIO		2

#define WMULT_IDLEPRIO		(1 << 31)
@@ -781,32 +728,6 @@ static const u32 prio_to_wmult[40] = { 
/*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,

};



static inline void

inc_load(struct rq *rq, const struct task_struct *p, u64 now)

{

	update_curr_load(rq, now);

	update_load_add(&rq->ls.load, p->se.load.weight);

}



static inline void

dec_load(struct rq *rq, const struct task_struct *p, u64 now)

{

	update_curr_load(rq, now);

	update_load_sub(&rq->ls.load, p->se.load.weight);

}



static inline void inc_nr_running(struct task_struct *p, struct rq *rq, u64 now)

{

	rq->nr_running++;

	inc_load(rq, p, now);

}



static inline void dec_nr_running(struct task_struct *p, struct rq *rq, u64 now)

{

	rq->nr_running--;

	dec_load(rq, p, now);

}



static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);



/*
@@ -837,6 +758,72 @@ static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, 


#define sched_class_highest (&rt_sched_class)



static void __update_curr_load(struct rq *rq, struct load_stat *ls)

{

	if (rq->curr != rq->idle && ls->load.weight) {

		ls->delta_exec += ls->delta_stat;

		ls->delta_fair += calc_delta_fair(ls->delta_stat, &ls->load);

		ls->delta_stat = 0;

	}

}



/*

 * Update delta_exec, delta_fair fields for rq.

 *

 * delta_fair clock advances at a rate inversely proportional to

 * total load (rq->ls.load.weight) on the runqueue, while

 * delta_exec advances at the same rate as wall-clock (provided

 * cpu is not idle).

 *

 * delta_exec / delta_fair is a measure of the (smoothened) load on this

 * runqueue over any given interval. This (smoothened) load is used

 * during load balance.

 *

 * This function is called /before/ updating rq->ls.load

 * and when switching tasks.

 */

static void update_curr_load(struct rq *rq, u64 now)

{

	struct load_stat *ls = &rq->ls;

	u64 start;



	start = ls->load_update_start;

	ls->load_update_start = now;

	ls->delta_stat += now - start;

	/*

	 * Stagger updates to ls->delta_fair. Very frequent updates

	 * can be expensive.

	 */

	if (ls->delta_stat >= sysctl_sched_stat_granularity)

		__update_curr_load(rq, ls);

}



static inline void

inc_load(struct rq *rq, const struct task_struct *p, u64 now)

{

	update_curr_load(rq, now);

	update_load_add(&rq->ls.load, p->se.load.weight);

}



static inline void

dec_load(struct rq *rq, const struct task_struct *p, u64 now)

{

	update_curr_load(rq, now);

	update_load_sub(&rq->ls.load, p->se.load.weight);

}



static void inc_nr_running(struct task_struct *p, struct rq *rq, u64 now)

{

	rq->nr_running++;

	inc_load(rq, p, now);

}



static void dec_nr_running(struct task_struct *p, struct rq *rq, u64 now)

{

	rq->nr_running--;

	dec_load(rq, p, now);

}



static void set_load_weight(struct task_struct *p)

{

	task_rq(p)->cfs.wait_runtime -= p->se.wait_runtime;
@@ -996,18 +983,21 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 
	u64 clock_offset, fair_clock_offset;



	clock_offset = old_rq->clock - new_rq->clock;

	fair_clock_offset = old_rq->cfs.fair_clock -

						 new_rq->cfs.fair_clock;

	if (p->se.wait_start)

		p->se.wait_start -= clock_offset;

	fair_clock_offset = old_rq->cfs.fair_clock - new_rq->cfs.fair_clock;



	if (p->se.wait_start_fair)

		p->se.wait_start_fair -= fair_clock_offset;

	if (p->se.sleep_start_fair)

		p->se.sleep_start_fair -= fair_clock_offset;



#ifdef CONFIG_SCHEDSTATS

	if (p->se.wait_start)

		p->se.wait_start -= clock_offset;

	if (p->se.sleep_start)

		p->se.sleep_start -= clock_offset;

	if (p->se.block_start)

		p->se.block_start -= clock_offset;

	if (p->se.sleep_start_fair)

		p->se.sleep_start_fair -= fair_clock_offset;

#endif



	__set_task_cpu(p, new_cpu);

}
@@ -1568,17 +1558,19 @@ int fastcall wake_up_state(struct task_struct *p, unsigned int state) 
static void __sched_fork(struct task_struct *p)

{

	p->se.wait_start_fair		= 0;

	p->se.wait_start		= 0;

	p->se.exec_start		= 0;

	p->se.sum_exec_runtime		= 0;

	p->se.delta_exec		= 0;

	p->se.delta_fair_run		= 0;

	p->se.delta_fair_sleep		= 0;

	p->se.wait_runtime		= 0;

	p->se.sleep_start_fair		= 0;



#ifdef CONFIG_SCHEDSTATS

	p->se.wait_start		= 0;

	p->se.sum_wait_runtime		= 0;

	p->se.sum_sleep_runtime		= 0;

	p->se.sleep_start		= 0;

	p->se.sleep_start_fair		= 0;

	p->se.block_start		= 0;

	p->se.sleep_max			= 0;

	p->se.block_max			= 0;
@@ -1586,6 +1578,7 @@ static void __sched_fork(struct task_struct *p) 
	p->se.wait_max			= 0;

	p->se.wait_runtime_overruns	= 0;

	p->se.wait_runtime_underruns	= 0;

#endif



	INIT_LIST_HEAD(&p->run_list);

	p->se.on_rq = 0;
@@ -1654,22 +1647,27 @@ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags) 
	unsigned long flags;

	struct rq *rq;

	int this_cpu;

	u64 now;



	rq = task_rq_lock(p, &flags);

	BUG_ON(p->state != TASK_RUNNING);

	this_cpu = smp_processor_id(); /* parent's CPU */

	now = rq_clock(rq);



	p->prio = effective_prio(p);



	if (!sysctl_sched_child_runs_first || (clone_flags & CLONE_VM) ||

			task_cpu(p) != this_cpu || !current->se.on_rq) {

	if (!p->sched_class->task_new || !sysctl_sched_child_runs_first ||

			(clone_flags & CLONE_VM) || task_cpu(p) != this_cpu ||

			!current->se.on_rq) {



		activate_task(rq, p, 0);

	} else {

		/*

		 * Let the scheduling class do new task startup

		 * management (if any):

		 */

		p->sched_class->task_new(rq, p);

		p->sched_class->task_new(rq, p, now);

		inc_nr_running(p, rq, now);

	}

	check_preempt_curr(rq, p);

	task_rq_unlock(rq, &flags);
@@ -2908,8 +2906,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu) 
		schedstat_inc(sd, alb_cnt);



		if (move_tasks(target_rq, target_cpu, busiest_rq, 1,

			       RTPRIO_TO_LOAD_WEIGHT(100), sd, CPU_IDLE,

			       NULL))

			       ULONG_MAX, sd, CPU_IDLE, NULL))

			schedstat_inc(sd, alb_pushed);

		else

			schedstat_inc(sd, alb_failed);
@@ -5269,8 +5266,6 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd) 
		sizeof(int), 0644, proc_dointvec_minmax);

	set_table_entry(&table[8], 9, "imbalance_pct", &sd->imbalance_pct,

		sizeof(int), 0644, proc_dointvec_minmax);

	set_table_entry(&table[9], 10, "cache_hot_time", &sd->cache_hot_time,

		sizeof(long long), 0644, proc_doulongvec_minmax);

	set_table_entry(&table[10], 11, "cache_nice_tries",

		&sd->cache_nice_tries,

		sizeof(int), 0644, proc_dointvec_minmax);
@@ -6590,12 +6585,14 @@ void normalize_rt_tasks(void) 
	do_each_thread(g, p) {

		p->se.fair_key			= 0;

		p->se.wait_runtime		= 0;

		p->se.exec_start		= 0;

		p->se.wait_start_fair		= 0;

		p->se.sleep_start_fair		= 0;

#ifdef CONFIG_SCHEDSTATS

		p->se.wait_start		= 0;

		p->se.exec_start		= 0;

		p->se.sleep_start		= 0;

		p->se.sleep_start_fair		= 0;

		p->se.block_start		= 0;

#endif

		task_rq(p)->cfs.fair_clock	= 0;

		task_rq(p)->clock		= 0;

kernel/sched_debug.c

+16 −6
Original line number Diff line number Diff line
@@ -44,11 +44,16 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p, u64 now) 
		(long long)p->se.wait_runtime,

		(long long)(p->nvcsw + p->nivcsw),

		p->prio,

#ifdef CONFIG_SCHEDSTATS

		(long long)p->se.sum_exec_runtime,

		(long long)p->se.sum_wait_runtime,

		(long long)p->se.sum_sleep_runtime,

		(long long)p->se.wait_runtime_overruns,

		(long long)p->se.wait_runtime_underruns);

		(long long)p->se.wait_runtime_underruns

#else

		0LL, 0LL, 0LL, 0LL, 0LL

#endif

	);

}



static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu, u64 now)
@@ -171,7 +176,7 @@ static int sched_debug_show(struct seq_file *m, void *v) 
	u64 now = ktime_to_ns(ktime_get());

	int cpu;



	SEQ_printf(m, "Sched Debug Version: v0.05, %s %.*s\n",

	SEQ_printf(m, "Sched Debug Version: v0.05-v20, %s %.*s\n",

		init_utsname()->release,

		(int)strcspn(init_utsname()->version, " "),

		init_utsname()->version);
@@ -235,21 +240,24 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) 
#define P(F) \

	SEQ_printf(m, "%-25s:%20Ld\n", #F, (long long)p->F)



	P(se.wait_start);

	P(se.wait_runtime);

	P(se.wait_start_fair);

	P(se.exec_start);

	P(se.sleep_start);

	P(se.sleep_start_fair);

	P(se.sum_exec_runtime);



#ifdef CONFIG_SCHEDSTATS

	P(se.wait_start);

	P(se.sleep_start);

	P(se.block_start);

	P(se.sleep_max);

	P(se.block_max);

	P(se.exec_max);

	P(se.wait_max);

	P(se.wait_runtime);

	P(se.wait_runtime_overruns);

	P(se.wait_runtime_underruns);

	P(se.sum_wait_runtime);

	P(se.sum_exec_runtime);

#endif

	SEQ_printf(m, "%-25s:%20Ld\n",

		   "nr_switches", (long long)(p->nvcsw + p->nivcsw));

	P(se.load.weight);
@@ -269,7 +277,9 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) 


void proc_sched_set_task(struct task_struct *p)

{

#ifdef CONFIG_SCHEDSTATS

	p->se.sleep_max = p->se.block_max = p->se.exec_max = p->se.wait_max = 0;

	p->se.wait_runtime_overruns = p->se.wait_runtime_underruns = 0;

#endif

	p->se.sum_exec_runtime = 0;

}

kernel/sched_fair.c

+5 −16
Original line number Diff line number Diff line
@@ -292,10 +292,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, u64 now) 
		return;



	delta_exec = curr->delta_exec;

#ifdef CONFIG_SCHEDSTATS

	if (unlikely(delta_exec > curr->exec_max))

		curr->exec_max = delta_exec;

#endif

	schedstat_set(curr->exec_max, max((u64)delta_exec, curr->exec_max));



	curr->sum_exec_runtime += delta_exec;

	cfs_rq->exec_clock += delta_exec;
@@ -352,7 +349,7 @@ static inline void 
update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)

{

	se->wait_start_fair = cfs_rq->fair_clock;

	se->wait_start = now;

	schedstat_set(se->wait_start, now);

}



/*
@@ -425,13 +422,7 @@ __update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 
{

	unsigned long delta_fair = se->delta_fair_run;



#ifdef CONFIG_SCHEDSTATS

	{

		s64 delta_wait = now - se->wait_start;

		if (unlikely(delta_wait > se->wait_max))

			se->wait_max = delta_wait;

	}

#endif

	schedstat_set(se->wait_max, max(se->wait_max, now - se->wait_start));



	if (unlikely(se->load.weight != NICE_0_LOAD))

		delta_fair = calc_weighted(delta_fair, se->load.weight,
@@ -456,7 +447,7 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 
	}



	se->wait_start_fair = 0;

	se->wait_start = 0;

	schedstat_set(se->wait_start, 0);

}



static inline void
@@ -1041,11 +1032,10 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr) 
 * monopolize the CPU. Note: the parent runqueue is locked,

 * the child is not running yet.

 */

static void task_new_fair(struct rq *rq, struct task_struct *p)

static void task_new_fair(struct rq *rq, struct task_struct *p, u64 now)

{

	struct cfs_rq *cfs_rq = task_cfs_rq(p);

	struct sched_entity *se = &p->se;

	u64 now = rq_clock(rq);



	sched_info_queued(p);
@@ -1072,7 +1062,6 @@ static void task_new_fair(struct rq *rq, struct task_struct *p) 
		p->se.wait_runtime = -(sysctl_sched_granularity / 2);



	__enqueue_entity(cfs_rq, se);

	inc_nr_running(p, rq, now);

}



#ifdef CONFIG_FAIR_GROUP_SCHED

CRA Git | Maintained and supported by SUSTech CRA and CCSE