#include #include #ifdef HAVE_ANDROID_GETCPUFEATURES # include #endif #include "private/common.h" #include "runtime.h" typedef struct CPUFeatures_ { int initialized; int has_neon; int has_sse2; int has_sse3; int has_ssse3; int has_sse41; int has_avx; int has_avx2; int has_avx512f; int has_pclmul; int has_aesni; int has_rdrand; } CPUFeatures; static CPUFeatures _cpu_features; #define CPUID_EBX_AVX2 0x00000020 #define CPUID_EBX_AVX512F 0x00010000 #define CPUID_ECX_SSE3 0x00000001 #define CPUID_ECX_PCLMUL 0x00000002 #define CPUID_ECX_SSSE3 0x00000200 #define CPUID_ECX_SSE41 0x00080000 #define CPUID_ECX_AESNI 0x02000000 #define CPUID_ECX_XSAVE 0x04000000 #define CPUID_ECX_OSXSAVE 0x08000000 #define CPUID_ECX_AVX 0x10000000 #define CPUID_ECX_RDRAND 0x40000000 #define CPUID_EDX_SSE2 0x04000000 #define XCR0_SSE 0x00000002 #define XCR0_AVX 0x00000004 static int _sodium_runtime_arm_cpu_features(CPUFeatures * const cpu_features) { #ifndef __arm__ cpu_features->has_neon = 0; return -1; #else # ifdef __APPLE__ # ifdef __ARM_NEON__ cpu_features->has_neon = 1; # else cpu_features->has_neon = 0; # endif # elif defined(HAVE_ANDROID_GETCPUFEATURES) && \ defined(ANDROID_CPU_ARM_FEATURE_NEON) cpu_features->has_neon = (android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0x0; # else cpu_features->has_neon = 0; # endif return 0; #endif } static void _cpuid(unsigned int cpu_info[4U], const unsigned int cpu_info_type) { #if defined(_MSC_VER) && \ (defined(_M_X64) || defined(_M_AMD64) || defined(_M_IX86)) __cpuid((int *) cpu_info, cpu_info_type); #elif defined(HAVE_CPUID) cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; # ifdef __i386__ __asm__ __volatile__( "pushfl; pushfl; " "popl %0; " "movl %0, %1; xorl %2, %0; " "pushl %0; " "popfl; pushfl; popl %0; popfl" : "=&r"(cpu_info[0]), "=&r"(cpu_info[1]) : "i"(0x200000)); if (((cpu_info[0] ^ cpu_info[1]) & 0x200000) == 0x0) { return; /* LCOV_EXCL_LINE */ } # endif # ifdef __i386__ __asm__ __volatile__("xchgl %%ebx, %k1; cpuid; xchgl %%ebx, %k1" : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) : "0"(cpu_info_type), "2"(0U)); # elif defined(__x86_64__) __asm__ __volatile__("xchgq %%rbx, %q1; cpuid; xchgq %%rbx, %q1" : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) : "0"(cpu_info_type), "2"(0U)); # else __asm__ __volatile__("cpuid" : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) : "0"(cpu_info_type), "2"(0U)); # endif #else (void) cpu_info_type; cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; #endif } static int _sodium_runtime_intel_cpu_features(CPUFeatures * const cpu_features) { unsigned int cpu_info[4]; unsigned int id; _cpuid(cpu_info, 0x0); if ((id = cpu_info[0]) == 0U) { return -1; /* LCOV_EXCL_LINE */ } _cpuid(cpu_info, 0x00000001); #ifdef HAVE_EMMINTRIN_H cpu_features->has_sse2 = ((cpu_info[3] & CPUID_EDX_SSE2) != 0x0); #else cpu_features->has_sse2 = 0; #endif #ifdef HAVE_PMMINTRIN_H cpu_features->has_sse3 = ((cpu_info[2] & CPUID_ECX_SSE3) != 0x0); #else cpu_features->has_sse3 = 0; #endif #ifdef HAVE_TMMINTRIN_H cpu_features->has_ssse3 = ((cpu_info[2] & CPUID_ECX_SSSE3) != 0x0); #else cpu_features->has_ssse3 = 0; #endif #ifdef HAVE_SMMINTRIN_H cpu_features->has_sse41 = ((cpu_info[2] & CPUID_ECX_SSE41) != 0x0); #else cpu_features->has_sse41 = 0; #endif cpu_features->has_avx = 0; #ifdef HAVE_AVXINTRIN_H if ((cpu_info[2] & (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) == (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) { uint32_t xcr0 = 0U; # if defined(HAVE__XGETBV) || \ (defined(_MSC_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) && _MSC_FULL_VER >= 160040219) xcr0 = (uint32_t) _xgetbv(0); # elif defined(_MSC_VER) && defined(_M_IX86) /* * Visual Studio documentation states that eax/ecx/edx don't need to * be preserved in inline assembly code. But that doesn't seem to * always hold true on Visual Studio 2010. */ __asm { push eax push ecx push edx xor ecx, ecx _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0 mov xcr0, eax pop edx pop ecx pop eax } # elif defined(HAVE_AVX_ASM) __asm__ __volatile__(".byte 0x0f, 0x01, 0xd0" /* XGETBV */ : "=a"(xcr0) : "c"((uint32_t) 0U) : "%edx"); # endif if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) { cpu_features->has_avx = 1; } } #endif cpu_features->has_avx2 = 0; #ifdef HAVE_AVX2INTRIN_H if (cpu_features->has_avx) { unsigned int cpu_info7[4]; _cpuid(cpu_info7, 0x00000007); cpu_features->has_avx2 = ((cpu_info7[1] & CPUID_EBX_AVX2) != 0x0); } #endif cpu_features->has_avx512f = 0; #ifdef HAVE_AVX512FINTRIN_H if (cpu_features->has_avx2) { unsigned int cpu_info7[4]; _cpuid(cpu_info7, 0x00000007); cpu_features->has_avx512f = ((cpu_info7[1] & CPUID_EBX_AVX512F) != 0x0); } #endif #ifdef HAVE_WMMINTRIN_H cpu_features->has_pclmul = ((cpu_info[2] & CPUID_ECX_PCLMUL) != 0x0); cpu_features->has_aesni = ((cpu_info[2] & CPUID_ECX_AESNI) != 0x0); #else cpu_features->has_pclmul = 0; cpu_features->has_aesni = 0; #endif #ifdef HAVE_RDRAND cpu_features->has_rdrand = ((cpu_info[2] & CPUID_ECX_RDRAND) != 0x0); #else cpu_features->has_rdrand = 0; #endif return 0; } int _sodium_runtime_get_cpu_features(void) { int ret = -1; ret &= _sodium_runtime_arm_cpu_features(&_cpu_features); ret &= _sodium_runtime_intel_cpu_features(&_cpu_features); _cpu_features.initialized = 1; return ret; } int sodium_runtime_has_neon(void) { return _cpu_features.has_neon; } int sodium_runtime_has_sse2(void) { return _cpu_features.has_sse2; } int sodium_runtime_has_sse3(void) { return _cpu_features.has_sse3; } int sodium_runtime_has_ssse3(void) { return _cpu_features.has_ssse3; } int sodium_runtime_has_sse41(void) { return _cpu_features.has_sse41; } int sodium_runtime_has_avx(void) { return _cpu_features.has_avx; } int sodium_runtime_has_avx2(void) { return _cpu_features.has_avx2; } int sodium_runtime_has_avx512f(void) { return _cpu_features.has_avx512f; } int sodium_runtime_has_pclmul(void) { return _cpu_features.has_pclmul; } int sodium_runtime_has_aesni(void) { return _cpu_features.has_aesni; } int sodium_runtime_has_rdrand(void) { return _cpu_features.has_rdrand; }