Commit d61931d8 authored by Borislav Petkov's avatar Borislav Petkov Committed by H. Peter Anvin
Browse files

x86: Add optimized popcnt variants 



Add support for the hardware version of the Hamming weight function,
popcnt, present in CPUs which advertize it under CPUID, Function
0x0000_0001_ECX[23]. On CPUs which don't support it, we fallback to the
default lib/hweight.c sw versions.

A synthetic benchmark comparing popcnt with __sw_hweight64 showed almost
a 3x speedup on a F10h machine.

Signed-off-by: default avatarBorislav Petkov <borislav.petkov@amd.com>
LKML-Reference: <20100318112015.GC11152@aftab>
Signed-off-by: default avatarH. Peter Anvin <hpa@zytor.com>
parent 1527bc8b

arch/x86/Kconfig

+5 −0
Original line number Diff line number Diff line
@@ -238,6 +238,11 @@ config X86_32_LAZY_GS 
	def_bool y

	depends on X86_32 && !CC_STACKPROTECTOR



config ARCH_HWEIGHT_CFLAGS

	string

	default "-fcall-saved-ecx -fcall-saved-edx" if X86_32

	default "-fcall-saved-rdi -fcall-saved-rsi -fcall-saved-rdx -fcall-saved-rcx -fcall-saved-r8 -fcall-saved-r9 -fcall-saved-r10 -fcall-saved-r11" if X86_64



config KTIME_SCALAR

	def_bool X86_32

source "init/Kconfig"

arch/x86/include/asm/alternative.h

+6 −3
Original line number Diff line number Diff line
@@ -39,9 +39,6 @@ 
#define LOCK_PREFIX ""

#endif



/* This must be included *after* the definition of LOCK_PREFIX */

#include <asm/cpufeature.h>



struct alt_instr {

	u8 *instr;		/* original instruction */

	u8 *replacement;
@@ -95,6 +92,12 @@ static inline int alternatives_text_reserved(void *start, void *end) 
      "663:\n\t" newinstr "\n664:\n"		/* replacement     */	\

      ".previous"



/*

 * This must be included *after* the definition of ALTERNATIVE due to

 * <asm/arch_hweight.h>

 */

#include <asm/cpufeature.h>



/*

 * Alternative instructions for different CPU types or capabilities.

 *

arch/x86/include/asm/arch_hweight.h

0 → 100644
+59 −0
Original line number Diff line number Diff line 
#ifndef _ASM_X86_HWEIGHT_H

#define _ASM_X86_HWEIGHT_H



#ifdef CONFIG_64BIT

/* popcnt %rdi, %rax */

#define POPCNT ".byte 0xf3,0x48,0x0f,0xb8,0xc7"

#define REG_IN "D"

#define REG_OUT "a"

#else

/* popcnt %eax, %eax */

#define POPCNT ".byte 0xf3,0x0f,0xb8,0xc0"

#define REG_IN "a"

#define REG_OUT "a"

#endif



/*

 * __sw_hweightXX are called from within the alternatives below

 * and callee-clobbered registers need to be taken care of. See

 * ARCH_HWEIGHT_CFLAGS in <arch/x86/Kconfig> for the respective

 * compiler switches.

 */

static inline unsigned int __arch_hweight32(unsigned int w)

{

	unsigned int res = 0;



	asm (ALTERNATIVE("call __sw_hweight32", POPCNT, X86_FEATURE_POPCNT)

		     : "="REG_OUT (res)

		     : REG_IN (w));



	return res;

}



static inline unsigned int __arch_hweight16(unsigned int w)

{

	return __arch_hweight32(w & 0xffff);

}



static inline unsigned int __arch_hweight8(unsigned int w)

{

	return __arch_hweight32(w & 0xff);

}



static inline unsigned long __arch_hweight64(__u64 w)

{

	unsigned long res = 0;



#ifdef CONFIG_X86_32

	return  __arch_hweight32((u32)w) +

		__arch_hweight32((u32)(w >> 32));

#else

	asm (ALTERNATIVE("call __sw_hweight64", POPCNT, X86_FEATURE_POPCNT)

		     : "="REG_OUT (res)

		     : REG_IN (w));

#endif /* CONFIG_X86_32 */



	return res;

}



#endif

arch/x86/include/asm/bitops.h

+3 −1
Original line number Diff line number Diff line
@@ -444,7 +444,9 @@ static inline int fls(int x) 


#define ARCH_HAS_FAST_MULTIPLIER 1



#include <asm-generic/bitops/hweight.h>

#include <asm/arch_hweight.h>



#include <asm-generic/bitops/const_hweight.h>



#endif /* __KERNEL__ */

include/asm-generic/bitops/arch_hweight.h

+18 −4
Original line number Diff line number Diff line
@@ -3,9 +3,23 @@ 


#include <asm/types.h>



extern unsigned int __arch_hweight32(unsigned int w);

extern unsigned int __arch_hweight16(unsigned int w);

extern unsigned int __arch_hweight8(unsigned int w);

extern unsigned long __arch_hweight64(__u64 w);

inline unsigned int __arch_hweight32(unsigned int w)

{

	return __sw_hweight32(w);

}



inline unsigned int __arch_hweight16(unsigned int w)

{

	return __sw_hweight16(w);

}



inline unsigned int __arch_hweight8(unsigned int w)

{

	return __sw_hweight8(w);

}



inline unsigned long __arch_hweight64(__u64 w)

{

	return __sw_hweight64(w);

}

#endif /* _ASM_GENERIC_BITOPS_HWEIGHT_H_ */

CRA Git | Maintained and supported by SUSTech CRA and CCSE