Commit 5fbda3ec authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

sched: highmem: Store local kmaps in task struct 



Instead of storing the map per CPU provide and use per task storage. That
prepares for local kmaps which are preemptible.

The context switch code is preparatory and not yet in use because
kmap_atomic() runs with preemption disabled. Will be made usable in the
next step.

The context switch logic is safe even when an interrupt happens after
clearing or before restoring the kmaps. The kmap index in task struct is
not modified so any nesting kmap in an interrupt will use unused indices
and on return the counter is the same as before.

Also add an assert into the return to user space code. Going back to user
space with an active kmap local is a nono.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20201118204007.372935758@linutronix.de
parent 14df3267

include/linux/highmem-internal.h

+10 −0
Original line number Diff line number Diff line
@@ -9,6 +9,16 @@ 
void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot);

void *__kmap_local_page_prot(struct page *page, pgprot_t prot);

void kunmap_local_indexed(void *vaddr);

void kmap_local_fork(struct task_struct *tsk);

void __kmap_local_sched_out(void);

void __kmap_local_sched_in(void);

static inline void kmap_assert_nomap(void)

{

	DEBUG_LOCKS_WARN_ON(current->kmap_ctrl.idx);

}

#else

static inline void kmap_local_fork(struct task_struct *tsk) { }

static inline void kmap_assert_nomap(void) { }

#endif



#ifdef CONFIG_HIGHMEM

include/linux/sched.h

+9 −0
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ 
#include <linux/rseq.h>

#include <linux/seqlock.h>

#include <linux/kcsan.h>

#include <asm/kmap_size.h>



/* task_struct member predeclarations (sorted alphabetically): */

struct audit_context;
@@ -629,6 +630,13 @@ struct wake_q_node { 
	struct wake_q_node *next;

};



struct kmap_ctrl {

#ifdef CONFIG_KMAP_LOCAL

	int				idx;

	pte_t				pteval[KM_MAX_IDX];

#endif

};



struct task_struct {

#ifdef CONFIG_THREAD_INFO_IN_TASK

	/*
@@ -1294,6 +1302,7 @@ struct task_struct { 
	unsigned int			sequential_io;

	unsigned int			sequential_io_avg;

#endif

	struct kmap_ctrl		kmap_ctrl;

#ifdef CONFIG_DEBUG_ATOMIC_SLEEP

	unsigned long			task_state_change;

#endif

kernel/entry/common.c

+2 −0
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ 


#include <linux/context_tracking.h>

#include <linux/entry-common.h>

#include <linux/highmem.h>

#include <linux/livepatch.h>

#include <linux/audit.h>
@@ -194,6 +195,7 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) 


	/* Ensure that the address limit is intact and no locks are held */

	addr_limit_user_check();

	kmap_assert_nomap();

	lockdep_assert_irqs_disabled();

	lockdep_sys_exit();

}

kernel/fork.c

+1 −0
Original line number Diff line number Diff line
@@ -930,6 +930,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) 
	account_kernel_stack(tsk, 1);



	kcov_task_init(tsk);

	kmap_local_fork(tsk);



#ifdef CONFIG_FAULT_INJECTION

	tsk->fail_nth = 0;

kernel/sched/core.c

+25 −0
Original line number Diff line number Diff line
@@ -4094,6 +4094,22 @@ static inline void finish_lock_switch(struct rq *rq) 
# define finish_arch_post_lock_switch()	do { } while (0)

#endif



static inline void kmap_local_sched_out(void)

{

#ifdef CONFIG_KMAP_LOCAL

	if (unlikely(current->kmap_ctrl.idx))

		__kmap_local_sched_out();

#endif

}



static inline void kmap_local_sched_in(void)

{

#ifdef CONFIG_KMAP_LOCAL

	if (unlikely(current->kmap_ctrl.idx))

		__kmap_local_sched_in();

#endif

}



/**

 * prepare_task_switch - prepare to switch tasks

 * @rq: the runqueue preparing to switch
@@ -4116,6 +4132,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, 
	perf_event_task_sched_out(prev, next);

	rseq_preempt(prev);

	fire_sched_out_preempt_notifiers(prev, next);

	kmap_local_sched_out();

	prepare_task(next);

	prepare_arch_switch(next);

}
@@ -4182,6 +4199,14 @@ static struct rq *finish_task_switch(struct task_struct *prev) 
	finish_lock_switch(rq);

	finish_arch_post_lock_switch();

	kcov_finish_switch(current);

	/*

	 * kmap_local_sched_out() is invoked with rq::lock held and

	 * interrupts disabled. There is no requirement for that, but the

	 * sched out code does not have an interrupt enabled section.

	 * Restoring the maps on sched in does not require interrupts being

	 * disabled either.

	 */

	kmap_local_sched_in();



	fire_sched_in_preempt_notifiers(current);

	/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE