diff --git a/makefile b/makefile index 43d9afb..54112f7 100644 --- a/makefile +++ b/makefile @@ -24,10 +24,14 @@ ${BIN}: ${OBJ} @echo [L] $@ @${CC} ${LDFLAGS} -o $@ $^ -%_x86.o: %_x86.c +%aes_x86.o: %aes_x86.c @echo [C] $< @${CC} ${CFLAGS} -maes -mssse3 -MMD -c -o $@ $< +%crc32_x86.o: %crc32_x86.c + @echo [C] $< + @${CC} ${CFLAGS} -mpclmul -msse4 -MMD -c -o $@ $< + %.o: %.c @echo [C] $< @${CC} ${CFLAGS} -MMD -c -o $@ $< diff --git a/pkg2zip_aes.h b/pkg2zip_aes.h index 4627f5a..c4acfbe 100644 --- a/pkg2zip_aes.h +++ b/pkg2zip_aes.h @@ -3,7 +3,7 @@ #include "pkg2zip_utils.h" typedef struct { - MSVC_ALIGN(16) uint32_t key[44] GCC_ALIGN(16); + uint32_t PKG_ALIGN(16) key[44]; } aes128_key; void aes128_init(aes128_key* context, const uint8_t* key); diff --git a/pkg2zip_crc32.c b/pkg2zip_crc32.c index 44e5756..27b95be 100644 --- a/pkg2zip_crc32.c +++ b/pkg2zip_crc32.c @@ -5,6 +5,55 @@ #include #endif +#if defined(_MSC_VER) +#define PLATFORM_SUPPORTS_PCLMUL 1 + +#include +static void get_cpuid(uint32_t level, uint32_t* arr) +{ + __cpuidex((int*)arr, level, 0); +} + +#elif defined(__x86_64__) || defined(__i386__) +#define PLATFORM_SUPPORTS_PCLMUL 1 + +#include +static void get_cpuid(uint32_t level, uint32_t* arr) +{ + __cpuid_count(level, 0, arr[0], arr[1], arr[2], arr[3]); +} + +#else +#define PLATFORM_SUPPORTS_PCLMUL 0 +#endif + +#if PLATFORM_SUPPORTS_PCLMUL +static int crc32_supported_x86() +{ + static int init = 0; + static int supported; + if (!init) + { + init = 1; + + uint32_t a[4]; + get_cpuid(0, a); + + if (a[0] >= 1) + { + get_cpuid(1, a); + supported = ((a[2] & (1 << 9)) && (a[2] & (1 << 1)) && (a[2] & (1 << 19))); + } + } + return supported; +} + +void crc32_init_x86(crc32_ctx* ctx); +void crc32_update_x86(crc32_ctx* ctx, const void* buffer, size_t size); +uint32_t crc32_done_x86(crc32_ctx* ctx); + +#endif + static const uint32_t crc32[4][256] = { { @@ -147,13 +196,28 @@ static const uint32_t crc32[4][256] = void crc32_init(crc32_ctx* ctx) { - ctx->crc = 0xffffffff; +#if PLATFORM_SUPPORTS_PCLMUL + if (crc32_supported_x86()) + { + crc32_init_x86(ctx); + return; + } +#endif + ctx->crc[0] = 0xffffffff; } void crc32_update(crc32_ctx* ctx, const void* buffer, size_t size) { +#if PLATFORM_SUPPORTS_PCLMUL + if (crc32_supported_x86()) + { + crc32_update_x86(ctx, buffer, size); + return; + } +#endif + const uint8_t* buffer8 = buffer; - uint32_t value = ctx->crc; + uint32_t value = ctx->crc[0]; while (size >= 4) { #if defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN @@ -180,10 +244,17 @@ void crc32_update(crc32_ctx* ctx, const void* buffer, size_t size) value = (value >> 8) ^ crc32[0][(uint8_t)value ^ *buffer8++]; } - ctx->crc = value; + ctx->crc[0] = value; } uint32_t crc32_done(crc32_ctx* ctx) { - return ~ctx->crc; +#if PLATFORM_SUPPORTS_PCLMUL + if (crc32_supported_x86()) + { + return crc32_done_x86(ctx); + } +#endif + + return ~ctx->crc[0]; } diff --git a/pkg2zip_crc32.h b/pkg2zip_crc32.h index c62fea1..977cc0e 100644 --- a/pkg2zip_crc32.h +++ b/pkg2zip_crc32.h @@ -3,7 +3,7 @@ #include "pkg2zip_utils.h" typedef struct { - MSVC_ALIGN(16) uint32_t crc GCC_ALIGN(16); + uint32_t PKG_ALIGN(16) crc[4 * 5]; } crc32_ctx; void crc32_init(crc32_ctx* ctx); diff --git a/pkg2zip_crc32_x86.c b/pkg2zip_crc32_x86.c new file mode 100644 index 0000000..c99a8c2 --- /dev/null +++ b/pkg2zip_crc32_x86.c @@ -0,0 +1,336 @@ +#include "pkg2zip_crc32.h" + +#include // PCLMUL +#include // SSSE3 +#include // SSS4 + +// Whitepaper: https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf +// ZLIB licensed code from https://github.com/jtkukunas/zlib/blob/master/crc_folding.c + +static const uint32_t PKG_ALIGN(16) shift_table[] = { + 0x84838281, 0x88878685, 0x8c8b8a89, 0x008f8e8d, + 0x85848382, 0x89888786, 0x8d8c8b8a, 0x01008f8e, + 0x86858483, 0x8a898887, 0x8e8d8c8b, 0x0201008f, + 0x87868584, 0x8b8a8988, 0x8f8e8d8c, 0x03020100, + 0x88878685, 0x8c8b8a89, 0x008f8e8d, 0x04030201, + 0x89888786, 0x8d8c8b8a, 0x01008f8e, 0x05040302, + 0x8a898887, 0x8e8d8c8b, 0x0201008f, 0x06050403, + 0x8b8a8988, 0x8f8e8d8c, 0x03020100, 0x07060504, + 0x8c8b8a89, 0x008f8e8d, 0x04030201, 0x08070605, + 0x8d8c8b8a, 0x01008f8e, 0x05040302, 0x09080706, + 0x8e8d8c8b, 0x0201008f, 0x06050403, 0x0a090807, + 0x8f8e8d8c, 0x03020100, 0x07060504, 0x0b0a0908, + 0x008f8e8d, 0x04030201, 0x08070605, 0x0c0b0a09, + 0x01008f8e, 0x05040302, 0x09080706, 0x0d0c0b0a, + 0x0201008f, 0x06050403, 0x0a090807, 0x0e0d0c0b, +}; + +#define FOLD1(xmm0, xmm1, xmm2, xmm3) do \ +{ \ + const __m128i fold4 = _mm_set_epi32( \ + 0x00000001, 0x54442bd4, \ + 0x00000001, 0xc6e41596); \ + \ + __m128i r0, r1, r2, r3, a, b; \ + \ + r0 = xmm1; \ + r1 = xmm2; \ + r2 = xmm3; \ + \ + a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \ + r3 = _mm_xor_si128(a, b); \ + \ + xmm0 = r0; \ + xmm1 = r1; \ + xmm2 = r2; \ + xmm3 = r3; \ +} while (0) + +#define FOLD2(xmm0, xmm1, xmm2, xmm3) do \ +{ \ + const __m128i fold4 = _mm_set_epi32( \ + 0x00000001, 0x54442bd4, \ + 0x00000001, 0xc6e41596); \ + \ + __m128i r0, r1, r2, r3, a, b; \ + \ + r0 = xmm2; \ + r1 = xmm3; \ + \ + a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \ + r2 = _mm_xor_si128(a, b); \ + \ + a = _mm_clmulepi64_si128(xmm1, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm1, fold4, 0x10); \ + r3 = _mm_xor_si128(a, b); \ + \ + xmm0 = r0; \ + xmm1 = r1; \ + xmm2 = r2; \ + xmm3 = r3; \ +} while (0) + +#define FOLD3(xmm0, xmm1, xmm2, xmm3) do \ +{ \ + const __m128i fold4 = _mm_set_epi32( \ + 0x00000001, 0x54442bd4, \ + 0x00000001, 0xc6e41596); \ + \ + __m128i r0, r1, r2, r3, a, b; \ + \ + r0 = xmm3; \ + \ + a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \ + r1 = _mm_xor_si128(a, b); \ + \ + a = _mm_clmulepi64_si128(xmm1, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm1, fold4, 0x10); \ + r2 = _mm_xor_si128(a, b); \ + \ + a = _mm_clmulepi64_si128(xmm2, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm2, fold4, 0x10); \ + r3 = _mm_xor_si128(a, b); \ + \ + xmm0 = r0; \ + xmm1 = r1; \ + xmm2 = r2; \ + xmm3 = r3; \ +} while (0) + +#define FOLD4(xmm0, xmm1, xmm2, xmm3) do \ +{ \ + const __m128i fold4 = _mm_set_epi32( \ + 0x00000001, 0x54442bd4, \ + 0x00000001, 0xc6e41596); \ + \ + __m128i a, b; \ + \ + a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \ + xmm0 = _mm_xor_si128(a, b); \ + \ + a = _mm_clmulepi64_si128(xmm1, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm1, fold4, 0x10); \ + xmm1 = _mm_xor_si128(a, b); \ + \ + a = _mm_clmulepi64_si128(xmm2, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm2, fold4, 0x10); \ + xmm2 = _mm_xor_si128(a, b); \ + \ + a = _mm_clmulepi64_si128(xmm3, fold4, 0x01); \ + b = _mm_clmulepi64_si128(xmm3, fold4, 0x10); \ + xmm3 = _mm_xor_si128(a, b); \ +} while (0) + +#define PARTIAL(len, xmm0, xmm1, xmm2, xmm3, xmm4) do \ +{ \ + const __m128i fold4 = _mm_set_epi32( \ + 0x00000001, 0x54442bd4, \ + 0x00000001, 0xc6e41596); \ + const __m128i mask = _mm_set1_epi32(0x80808080); \ + \ + __m128i shl = _mm_load_si128((__m128i *)shift_table + (len - 1)); \ + __m128i shr = _mm_xor_si128(shl, mask); \ + \ + __m128i a, b, r; \ + __m128i tmp = _mm_shuffle_epi8(xmm0, shl); \ + \ + a = _mm_shuffle_epi8(xmm0, shr); \ + b = _mm_shuffle_epi8(xmm1, shl); \ + xmm0 = _mm_or_si128(a, b); \ + \ + a = _mm_shuffle_epi8(xmm1, shr); \ + b = _mm_shuffle_epi8(xmm2, shl); \ + xmm1 = _mm_or_si128(a, b); \ + \ + a = _mm_shuffle_epi8(xmm2, shr); \ + b = _mm_shuffle_epi8(xmm3, shl); \ + xmm2 = _mm_or_si128(a, b); \ + \ + a = _mm_shuffle_epi8(xmm3, shr); \ + b = _mm_shuffle_epi8(xmm4, shl); \ + xmm4 = b; \ + r = _mm_or_si128(a, b); \ + \ + a = _mm_clmulepi64_si128(tmp, fold4, 0x10); \ + b = _mm_clmulepi64_si128(tmp, fold4, 0x01); \ + \ + r = _mm_xor_si128(r, a); \ + r = _mm_xor_si128(r, b); \ + xmm3 = r; \ +} while(0) + +void crc32_init_x86(crc32_ctx* ctx) +{ + __m128i init = _mm_cvtsi32_si128(0x9db42487); + __m128i zero = _mm_setzero_si128(); + + _mm_store_si128((__m128i*)ctx->crc + 0, init); + _mm_store_si128((__m128i*)ctx->crc + 1, zero); + _mm_store_si128((__m128i*)ctx->crc + 2, zero); + _mm_store_si128((__m128i*)ctx->crc + 3, zero); +} + +void crc32_update_x86(crc32_ctx* ctx, const void* buffer, size_t size) +{ + const uint8_t* buffer8 = buffer; + + __m128i xmm0 = _mm_load_si128((__m128i*)ctx->crc + 0); + __m128i xmm1 = _mm_load_si128((__m128i*)ctx->crc + 1); + __m128i xmm2 = _mm_load_si128((__m128i*)ctx->crc + 2); + __m128i xmm3 = _mm_load_si128((__m128i*)ctx->crc + 3); + __m128i xmm4 = _mm_load_si128((__m128i*)ctx->crc + 4); + + if (size < 16) + { + if (size == 0) + { + return; + } + xmm4 = _mm_loadu_si128((__m128i *)buffer8); + goto partial; + } + + uint32_t prefix = (0 - (uintptr_t)buffer8) & 0xF; + if (prefix != 0) + { + xmm4 = _mm_loadu_si128((__m128i *)buffer8); + buffer8 += prefix; + size -= prefix; + + PARTIAL(prefix, xmm0, xmm1, xmm2, xmm3, xmm4); + } + + while (size >= 64) + { + __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0); + __m128i t1 = _mm_load_si128((__m128i *)buffer8 + 1); + __m128i t2 = _mm_load_si128((__m128i *)buffer8 + 2); + __m128i t3 = _mm_load_si128((__m128i *)buffer8 + 3); + + FOLD4(xmm0, xmm1, xmm2, xmm3); + + xmm0 = _mm_xor_si128(xmm0, t0); + xmm1 = _mm_xor_si128(xmm1, t1); + xmm2 = _mm_xor_si128(xmm2, t2); + xmm3 = _mm_xor_si128(xmm3, t3); + + buffer8 += 64; + size -= 64; + } + + if (size >= 48) + { + __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0); + __m128i t1 = _mm_load_si128((__m128i *)buffer8 + 1); + __m128i t2 = _mm_load_si128((__m128i *)buffer8 + 2); + + FOLD3(xmm0, xmm1, xmm2, xmm3); + + xmm1 = _mm_xor_si128(xmm1, t0); + xmm2 = _mm_xor_si128(xmm2, t1); + xmm3 = _mm_xor_si128(xmm3, t2); + + buffer8 += 48; + size -= 48; + } + else if (size >= 32) + { + __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0); + __m128i t1 = _mm_load_si128((__m128i *)buffer8 + 1); + + FOLD2(xmm0, xmm1, xmm2, xmm3); + + xmm2 = _mm_xor_si128(xmm2, t0); + xmm3 = _mm_xor_si128(xmm3, t1); + + buffer8 += 32; + size -= 32; + } + else if (size >= 16) + { + __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0); + + FOLD1(xmm0, xmm1, xmm2, xmm3); + + xmm3 = _mm_xor_si128(xmm3, t0); + + buffer8 += 16; + size -= 16; + } + + if (size == 0) + { + goto done; + } + + xmm4 = _mm_load_si128((__m128i *)buffer8); + +partial: + PARTIAL(size, xmm0, xmm1, xmm2, xmm3, xmm4); + +done: + _mm_store_si128((__m128i*)ctx->crc + 0, xmm0); + _mm_store_si128((__m128i*)ctx->crc + 1, xmm1); + _mm_store_si128((__m128i*)ctx->crc + 2, xmm2); + _mm_store_si128((__m128i*)ctx->crc + 3, xmm3); + _mm_store_si128((__m128i*)ctx->crc + 4, xmm4); +} + +uint32_t crc32_done_x86(crc32_ctx* ctx) +{ + const __m128i mask1 = _mm_setr_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000); + const __m128i mask2 = _mm_setr_epi32(0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); + + __m128i xmm0 = _mm_load_si128((__m128i*)ctx->crc + 0); + __m128i xmm1 = _mm_load_si128((__m128i*)ctx->crc + 1); + __m128i xmm2 = _mm_load_si128((__m128i*)ctx->crc + 2); + __m128i xmm3 = _mm_load_si128((__m128i*)ctx->crc + 3); + + __m128i fold; + __m128i a, b, t; + + fold = _mm_setr_epi32(0xccaa009e, 0x00000000, 0x751997d0, 0x00000001); + + a = _mm_clmulepi64_si128(xmm0, fold, 0x10); + b = _mm_clmulepi64_si128(xmm0, fold, 0x01); + xmm1 = _mm_xor_si128(xmm1, a); + xmm1 = _mm_xor_si128(xmm1, b); + + a = _mm_clmulepi64_si128(xmm1, fold, 0x10); + b = _mm_clmulepi64_si128(xmm1, fold, 0x01); + xmm2 = _mm_xor_si128(xmm2, a); + xmm2 = _mm_xor_si128(xmm2, b); + + a = _mm_clmulepi64_si128(xmm2, fold, 0x10); + b = _mm_clmulepi64_si128(xmm2, fold, 0x01); + xmm3 = _mm_xor_si128(xmm3, a); + xmm3 = _mm_xor_si128(xmm3, b); + + fold = _mm_setr_epi32(0xccaa009e, 0x00000000, 0x63cd6124, 0x00000001); + + a = _mm_clmulepi64_si128(xmm3, fold, 0); + b = _mm_srli_si128(xmm3, 8); + a = _mm_xor_si128(a, b); + + b = _mm_slli_si128(a, 4); + b = _mm_clmulepi64_si128(b, fold, 0x10); + t = _mm_xor_si128(a, b); + t = _mm_and_si128(t, mask2); + + fold = _mm_setr_epi32(0xf7011640, 0x00000001, 0xdb710640, 0x00000001); + + a = _mm_clmulepi64_si128(t, fold, 0); + a = _mm_xor_si128(a, t); + a = _mm_and_si128(a, mask1); + + b = _mm_clmulepi64_si128(a, fold, 0x10); + b = _mm_xor_si128(b, a); + b = _mm_xor_si128(b, t); + + uint32_t crc = _mm_extract_epi32(b, 2); + return ~crc; +} diff --git a/pkg2zip_utils.h b/pkg2zip_utils.h index 08b3197..f11444e 100644 --- a/pkg2zip_utils.h +++ b/pkg2zip_utils.h @@ -5,16 +5,10 @@ #if defined(_MSC_VER) # define NORETURN __declspec(noreturn) +# define PKG_ALIGN(x) __declspec(align(x)) #else # define NORETURN __attribute__((noreturn)) -#endif - -#if defined(_MSC_VER) -# define MSVC_ALIGN(x) __declspec(align(x)) -# define GCC_ALIGN(x) -#else -# define MSVC_ALIGN(x) -# define GCC_ALIGN(x) __attribute__((aligned(x))) +# define PKG_ALIGN(x) __attribute__((aligned(x))) #endif void NORETURN fatal(const char* msg, ...);