Commit eb3436a0 authored by Kumar Gala's avatar Kumar Gala Committed by Benjamin Herrenschmidt
Browse files

powerpc/mm: Used free register to save a few cycles in SW TLB miss handling 



Now that r0 is free we can keep the value of I/DMISS in r3 and not reload
it before doing the tlbli/d.  This saves us a few cycles in the fast path
case.

Signed-off-by: default avatarKumar Gala <galak@kernel.crashing.org>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent 00fcb147

arch/powerpc/kernel/head_32.S

+24 −27
Original line number Original line Diff line number Diff line
@@ -498,28 +498,27 @@ InstructionTLBMiss: 
	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */

	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */

	beq-	InstructionAddressInvalid	/* return if no mapping */

	beq-	InstructionAddressInvalid	/* return if no mapping */

	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */

	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */

	lwz	r3,0(r2)		/* get linux-style pte */

	lwz	r0,0(r2)		/* get linux-style pte */

	andc.	r1,r1,r3		/* check access & ~permission */

	andc.	r1,r1,r0		/* check access & ~permission */

	bne-	InstructionAddressInvalid /* return if access not permitted */

	bne-	InstructionAddressInvalid /* return if access not permitted */

	ori	r3,r3,_PAGE_ACCESSED	/* set _PAGE_ACCESSED in pte */

	ori	r0,r0,_PAGE_ACCESSED	/* set _PAGE_ACCESSED in pte */

	/*

	/*

	 * NOTE! We are assuming this is not an SMP system, otherwise

	 * NOTE! We are assuming this is not an SMP system, otherwise

	 * we would need to update the pte atomically with lwarx/stwcx.

	 * we would need to update the pte atomically with lwarx/stwcx.

	 */

	 */

	stw	r3,0(r2)		/* update PTE (accessed bit) */

	stw	r0,0(r2)		/* update PTE (accessed bit) */

	/* Convert linux-style PTE to low word of PPC-style PTE */

	/* Convert linux-style PTE to low word of PPC-style PTE */

	rlwinm	r1,r3,32-10,31,31	/* _PAGE_RW -> PP lsb */

	rlwinm	r1,r0,32-10,31,31	/* _PAGE_RW -> PP lsb */

	rlwinm	r2,r3,32-7,31,31	/* _PAGE_DIRTY -> PP lsb */

	rlwinm	r2,r0,32-7,31,31	/* _PAGE_DIRTY -> PP lsb */

	and	r1,r1,r2		/* writable if _RW and _DIRTY */

	and	r1,r1,r2		/* writable if _RW and _DIRTY */

	rlwimi	r3,r3,32-1,30,30	/* _PAGE_USER -> PP msb */

	rlwimi	r0,r0,32-1,30,30	/* _PAGE_USER -> PP msb */

	rlwimi	r3,r3,32-1,31,31	/* _PAGE_USER -> PP lsb */

	rlwimi	r0,r0,32-1,31,31	/* _PAGE_USER -> PP lsb */

	ori	r1,r1,0xe04		/* clear out reserved bits */

	ori	r1,r1,0xe04		/* clear out reserved bits */

	andc	r1,r3,r1		/* PP = user? (rw&dirty? 2: 3): 0 */

	andc	r1,r0,r1		/* PP = user? (rw&dirty? 2: 3): 0 */

BEGIN_FTR_SECTION

BEGIN_FTR_SECTION

	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */

	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */

END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)

END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)

	mtspr	SPRN_RPA,r1

	mtspr	SPRN_RPA,r1

	mfspr	r3,SPRN_IMISS

	tlbli	r3

	tlbli	r3

	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */

	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */

	mtcrf	0x80,r3

	mtcrf	0x80,r3
@@ -573,28 +572,27 @@ DataLoadTLBMiss: 
	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */

	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */

	beq-	DataAddressInvalid	/* return if no mapping */

	beq-	DataAddressInvalid	/* return if no mapping */

	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */

	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */

	lwz	r3,0(r2)		/* get linux-style pte */

	lwz	r0,0(r2)		/* get linux-style pte */

	andc.	r1,r1,r3		/* check access & ~permission */

	andc.	r1,r1,r0		/* check access & ~permission */

	bne-	DataAddressInvalid	/* return if access not permitted */

	bne-	DataAddressInvalid	/* return if access not permitted */

	ori	r3,r3,_PAGE_ACCESSED	/* set _PAGE_ACCESSED in pte */

	ori	r0,r0,_PAGE_ACCESSED	/* set _PAGE_ACCESSED in pte */

	/*

	/*

	 * NOTE! We are assuming this is not an SMP system, otherwise

	 * NOTE! We are assuming this is not an SMP system, otherwise

	 * we would need to update the pte atomically with lwarx/stwcx.

	 * we would need to update the pte atomically with lwarx/stwcx.

	 */

	 */

	stw	r3,0(r2)		/* update PTE (accessed bit) */

	stw	r0,0(r2)		/* update PTE (accessed bit) */

	/* Convert linux-style PTE to low word of PPC-style PTE */

	/* Convert linux-style PTE to low word of PPC-style PTE */

	rlwinm	r1,r3,32-10,31,31	/* _PAGE_RW -> PP lsb */

	rlwinm	r1,r0,32-10,31,31	/* _PAGE_RW -> PP lsb */

	rlwinm	r2,r3,32-7,31,31	/* _PAGE_DIRTY -> PP lsb */

	rlwinm	r2,r0,32-7,31,31	/* _PAGE_DIRTY -> PP lsb */

	and	r1,r1,r2		/* writable if _RW and _DIRTY */

	and	r1,r1,r2		/* writable if _RW and _DIRTY */

	rlwimi	r3,r3,32-1,30,30	/* _PAGE_USER -> PP msb */

	rlwimi	r0,r0,32-1,30,30	/* _PAGE_USER -> PP msb */

	rlwimi	r3,r3,32-1,31,31	/* _PAGE_USER -> PP lsb */

	rlwimi	r0,r0,32-1,31,31	/* _PAGE_USER -> PP lsb */

	ori	r1,r1,0xe04		/* clear out reserved bits */

	ori	r1,r1,0xe04		/* clear out reserved bits */

	andc	r1,r3,r1		/* PP = user? (rw&dirty? 2: 3): 0 */

	andc	r1,r0,r1		/* PP = user? (rw&dirty? 2: 3): 0 */

BEGIN_FTR_SECTION

BEGIN_FTR_SECTION

	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */

	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */

END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)

END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)

	mtspr	SPRN_RPA,r1

	mtspr	SPRN_RPA,r1

	mfspr	r3,SPRN_DMISS

	tlbld	r3

	tlbld	r3

	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */

	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */

	mtcrf	0x80,r3

	mtcrf	0x80,r3
@@ -646,24 +644,23 @@ DataStoreTLBMiss: 
	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */

	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */

	beq-	DataAddressInvalid	/* return if no mapping */

	beq-	DataAddressInvalid	/* return if no mapping */

	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */

	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */

	lwz	r3,0(r2)		/* get linux-style pte */

	lwz	r0,0(r2)		/* get linux-style pte */

	andc.	r1,r1,r3		/* check access & ~permission */

	andc.	r1,r1,r0		/* check access & ~permission */

	bne-	DataAddressInvalid	/* return if access not permitted */

	bne-	DataAddressInvalid	/* return if access not permitted */

	ori	r3,r3,_PAGE_ACCESSED|_PAGE_DIRTY

	ori	r0,r0,_PAGE_ACCESSED|_PAGE_DIRTY

	/*

	/*

	 * NOTE! We are assuming this is not an SMP system, otherwise

	 * NOTE! We are assuming this is not an SMP system, otherwise

	 * we would need to update the pte atomically with lwarx/stwcx.

	 * we would need to update the pte atomically with lwarx/stwcx.

	 */

	 */

	stw	r3,0(r2)		/* update PTE (accessed/dirty bits) */

	stw	r0,0(r2)		/* update PTE (accessed/dirty bits) */

	/* Convert linux-style PTE to low word of PPC-style PTE */

	/* Convert linux-style PTE to low word of PPC-style PTE */

	rlwimi	r3,r3,32-1,30,30	/* _PAGE_USER -> PP msb */

	rlwimi	r0,r0,32-1,30,30	/* _PAGE_USER -> PP msb */

	li	r1,0xe05		/* clear out reserved bits & PP lsb */

	li	r1,0xe05		/* clear out reserved bits & PP lsb */

	andc	r1,r3,r1		/* PP = user? 2: 0 */

	andc	r1,r0,r1		/* PP = user? 2: 0 */

BEGIN_FTR_SECTION

BEGIN_FTR_SECTION

	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */

	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */

END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)

END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)

	mtspr	SPRN_RPA,r1

	mtspr	SPRN_RPA,r1

	mfspr	r3,SPRN_DMISS

	tlbld	r3

	tlbld	r3

	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */

	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */

	mtcrf	0x80,r3

	mtcrf	0x80,r3

CRA Git | Maintained and supported by SUSTech CRA and CCSE