x86-64: Move TLB state from PDA to per-cpu and consolidate with 32-bit. (9eb912d1) · Commits · 戴 / test

arch/x86/include/asm/mmu_context_64.h

+7 −9

Original line number	Diff line number	Diff line
		#ifndef _ASM_X86_MMU_CONTEXT_64_H
		#define _ASM_X86_MMU_CONTEXT_64_H

		#include <asm/pda.h>

		static inline void enter_lazy_tlb(struct mm_struct mm, struct task_struct tsk)
		{
		#ifdef CONFIG_SMP
		if (read_pda(mmu_state) == TLBSTATE_OK)
		write_pda(mmu_state, TLBSTATE_LAZY);
		if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
		percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
		#endif
		}

		@@ -19,8 +17,8 @@ static inline void switch_mm(struct mm_struct prev, struct mm_struct next,
		/* stop flush ipis for the previous mm */
		cpu_clear(cpu, prev->cpu_vm_mask);
		#ifdef CONFIG_SMP
		write_pda(mmu_state, TLBSTATE_OK);
		write_pda(active_mm, next);
		percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
		percpu_write(cpu_tlbstate.active_mm, next);
		#endif
		cpu_set(cpu, next->cpu_vm_mask);
		load_cr3(next->pgd);
		@@ -30,9 +28,9 @@ static inline void switch_mm(struct mm_struct prev, struct mm_struct next,
		}
		#ifdef CONFIG_SMP
		else {
		write_pda(mmu_state, TLBSTATE_OK);
		if (read_pda(active_mm) != next)
		BUG();
		percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
		BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);

		if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
		/* We were in lazy tlb mode and leave_mm disabled
		* tlb flush IPI delivery. We must reload CR3

+0 −2

Original line number	Diff line number	Diff line
		@@ -25,9 +25,7 @@ struct x8664_pda {
		char *irqstackptr;
		short nodenumber; /* number of current node (32k max) */
		short in_bootmem; /* pda lives in bootmem */
		short mmu_state;
		short isidle;
		struct mm_struct *active_mm;
		} ____cacheline_aligned_in_smp;

		DECLARE_PER_CPU(struct x8664_pda, __pda);

+2 −5

Original line number	Diff line number	Diff line
		@@ -148,20 +148,17 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
		#define TLBSTATE_OK 1
		#define TLBSTATE_LAZY 2

		#ifdef CONFIG_X86_32
		struct tlb_state {
		struct mm_struct *active_mm;
		int state;
		char __cacheline_padding[L1_CACHE_BYTES-8];
		};
		DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);

		void reset_lazy_tlbstate(void);
		#else
		static inline void reset_lazy_tlbstate(void)
		{
		percpu_write(cpu_tlbstate.state, 0);
		percpu_write(cpu_tlbstate.active_mm, &init_mm);
		}
		#endif

		#endif /* SMP */

+0 −2

Original line number	Diff line number	Diff line
		@@ -897,8 +897,6 @@ void __cpuinit pda_init(int cpu)
		pda->irqcount = -1;
		pda->kernelstack = (unsigned long)stack_thread_info() -
		PDA_STACKOFFSET + THREAD_SIZE;
		pda->active_mm = &init_mm;
		pda->mmu_state = 0;

		if (cpu == 0) {
		/* others are initialized in smpboot.c */

+2 −10

Original line number	Diff line number	Diff line
		@@ -4,8 +4,8 @@

		#include <asm/tlbflush.h>

		DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)
		____cacheline_aligned = { &init_mm, 0, };
		DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
		= { &init_mm, 0, };

		/* must come after the send_IPI functions above for inlining */
		#include <mach_ipi.h>
		@@ -231,14 +231,6 @@ void flush_tlb_all(void)
		on_each_cpu(do_flush_tlb_all, NULL, 1);
		}

		void reset_lazy_tlbstate(void)
		{
		int cpu = raw_smp_processor_id();

		per_cpu(cpu_tlbstate, cpu).state = 0;
		per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;
		}

		static int init_flush_cpumask(void)
		{
		alloc_cpumask_var(&flush_cpumask, GFP_KERNEL);