Commit 9f79991d authored by Jeremy Fitzhardinge's avatar Jeremy Fitzhardinge Committed by Jeremy Fitzhardinge
Browse files

xen: deal with stale cr3 values when unpinning pagetables 



When a pagetable is no longer in use, it must be unpinned so that its
pages can be freed.  However, this is only possible if there are no
stray uses of the pagetable.  The code currently deals with all the
usual cases, but there's a rare case where a vcpu is changing cr3, but
is doing so lazily, and the change hasn't actually happened by the time
the pagetable is unpinned, even though it appears to have been completed.

This change adds a second per-cpu cr3 variable - xen_current_cr3 -
which tracks the actual state of the vcpu cr3.  It is only updated once
the actual hypercall to set cr3 has been completed.  Other processors
wishing to unpin a pagetable can check other vcpu's xen_current_cr3
values to see if any cross-cpu IPIs are needed to clean things up.

[ Stable folks: 2.6.23 bugfix ]

Signed-off-by: default avatarJeremy Fitzhardinge <jeremy@xensource.com>
Cc: Stable Kernel <stable@kernel.org>
parent 91e0c5f3

arch/x86/xen/enlighten.c

+38 −17
Original line number Diff line number Diff line
@@ -53,7 +53,23 @@ EXPORT_SYMBOL_GPL(hypercall_page); 


DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);

DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);

DEFINE_PER_CPU(unsigned long, xen_cr3);



/*

 * Note about cr3 (pagetable base) values:

 *

 * xen_cr3 contains the current logical cr3 value; it contains the

 * last set cr3.  This may not be the current effective cr3, because

 * its update may be being lazily deferred.  However, a vcpu looking

 * at its own cr3 can use this value knowing that it everything will

 * be self-consistent.

 *

 * xen_current_cr3 contains the actual vcpu cr3; it is set once the

 * hypercall to set the vcpu cr3 is complete (so it may be a little

 * out of date, but it will never be set early).  If one vcpu is

 * looking at another vcpu's cr3 value, it should use this variable.

 */

DEFINE_PER_CPU(unsigned long, xen_cr3);	 /* cr3 stored as physaddr */

DEFINE_PER_CPU(unsigned long, xen_current_cr3);	 /* actual vcpu cr3 */



struct start_info *xen_start_info;

EXPORT_SYMBOL_GPL(xen_start_info);
@@ -610,32 +626,36 @@ static unsigned long xen_read_cr3(void) 
	return x86_read_percpu(xen_cr3);

}



static void xen_write_cr3(unsigned long cr3)

static void set_current_cr3(void *v)

{

	BUG_ON(preemptible());



	if (cr3 == x86_read_percpu(xen_cr3)) {

		/* just a simple tlb flush */

		xen_flush_tlb();

		return;

	x86_write_percpu(xen_current_cr3, (unsigned long)v);

}



	x86_write_percpu(xen_cr3, cr3);





static void xen_write_cr3(unsigned long cr3)

{

	struct mmuext_op *op;

		struct multicall_space mcs = xen_mc_entry(sizeof(*op));

	struct multicall_space mcs;

	unsigned long mfn = pfn_to_mfn(PFN_DOWN(cr3));



	BUG_ON(preemptible());



	mcs = xen_mc_entry(sizeof(*op));  /* disables interrupts */



	/* Update while interrupts are disabled, so its atomic with

	   respect to ipis */

	x86_write_percpu(xen_cr3, cr3);



	op = mcs.args;

	op->cmd = MMUEXT_NEW_BASEPTR;

	op->arg1.mfn = mfn;



	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);



		xen_mc_issue(PARAVIRT_LAZY_CPU);

	}

	/* Update xen_update_cr3 once the batch has actually

	   been submitted. */

	xen_mc_callback(set_current_cr3, (void *)cr3);



	xen_mc_issue(PARAVIRT_LAZY_CPU);  /* interrupts restored */

}



/* Early in boot, while setting up the initial pagetable, assume
@@ -1120,6 +1140,7 @@ asmlinkage void __init xen_start_kernel(void) 
	/* keep using Xen gdt for now; no urgent need to change it */



	x86_write_percpu(xen_cr3, __pa(pgd));

	x86_write_percpu(xen_current_cr3, __pa(pgd));



#ifdef CONFIG_SMP

	/* Don't do the full vcpu_info placement stuff until we have a

arch/x86/xen/mmu.c

+26 −3
Original line number Diff line number Diff line
@@ -514,20 +514,43 @@ static void drop_other_mm_ref(void *info) 


	if (__get_cpu_var(cpu_tlbstate).active_mm == mm)

		leave_mm(smp_processor_id());



	/* If this cpu still has a stale cr3 reference, then make sure

	   it has been flushed. */

	if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {

		load_cr3(swapper_pg_dir);

		arch_flush_lazy_cpu_mode();

	}

}



static void drop_mm_ref(struct mm_struct *mm)

{

	cpumask_t mask;

	unsigned cpu;



	if (current->active_mm == mm) {

		if (current->mm == mm)

			load_cr3(swapper_pg_dir);

		else

			leave_mm(smp_processor_id());

		arch_flush_lazy_cpu_mode();

	}



	/* Get the "official" set of cpus referring to our pagetable. */

	mask = mm->cpu_vm_mask;



	/* It's possible that a vcpu may have a stale reference to our

	   cr3, because its in lazy mode, and it hasn't yet flushed

	   its set of pending hypercalls yet.  In this case, we can

	   look at its actual current cr3 value, and force it to flush

	   if needed. */

	for_each_online_cpu(cpu) {

		if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))

			cpu_set(cpu, mask);

	}



	if (!cpus_empty(mm->cpu_vm_mask))

		xen_smp_call_function_mask(mm->cpu_vm_mask, drop_other_mm_ref,

					   mm, 1);

	if (!cpus_empty(mask))

		xen_smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);

}

#else

static void drop_mm_ref(struct mm_struct *mm)

arch/x86/xen/xen-ops.h

+1 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ void xen_copy_trap_info(struct trap_info *traps); 


DECLARE_PER_CPU(struct vcpu_info *, xen_vcpu);

DECLARE_PER_CPU(unsigned long, xen_cr3);

DECLARE_PER_CPU(unsigned long, xen_current_cr3);



extern struct start_info *xen_start_info;

extern struct shared_info *HYPERVISOR_shared_info;

CRA Git | Maintained and supported by SUSTech CRA and CCSE