accelerated crc32 with PCLMUL instruction

makefile

@@ -24,10 +24,14 @@ ${BIN}: ${OBJ}
 	@echo [L] $@
 	@${CC} ${LDFLAGS} -o $@ $^
 
-%_x86.o: %_x86.c
+%aes_x86.o: %aes_x86.c
 	@echo [C] $<
 	@${CC} ${CFLAGS} -maes -mssse3 -MMD -c -o $@ $<
 
+%crc32_x86.o: %crc32_x86.c
+	@echo [C] $<
+	@${CC} ${CFLAGS} -mpclmul -msse4 -MMD -c -o $@ $<
+
 %.o: %.c
 	@echo [C] $<
 	@${CC} ${CFLAGS} -MMD -c -o $@ $<

pkg2zip_aes.h

@@ -3,7 +3,7 @@
 #include "pkg2zip_utils.h"
 
 typedef struct {
-    MSVC_ALIGN(16) uint32_t key[44] GCC_ALIGN(16);
+    uint32_t PKG_ALIGN(16) key[44];
 } aes128_key;
 
 void aes128_init(aes128_key* context, const uint8_t* key);

pkg2zip_crc32.c

@@ -5,6 +5,55 @@
 #include <endian.h>
 #endif
 
+#if defined(_MSC_VER)
+#define PLATFORM_SUPPORTS_PCLMUL 1
+
+#include <intrin.h>
+static void get_cpuid(uint32_t level, uint32_t* arr)
+{
+    __cpuidex((int*)arr, level, 0);
+}
+
+#elif defined(__x86_64__) || defined(__i386__)
+#define PLATFORM_SUPPORTS_PCLMUL 1
+
+#include <cpuid.h>
+static void get_cpuid(uint32_t level, uint32_t* arr)
+{
+    __cpuid_count(level, 0, arr[0], arr[1], arr[2], arr[3]);
+}
+
+#else
+#define PLATFORM_SUPPORTS_PCLMUL 0
+#endif
+
+#if PLATFORM_SUPPORTS_PCLMUL
+static int crc32_supported_x86()
+{
+    static int init = 0;
+    static int supported;
+    if (!init)
+    {
+        init = 1;
+
+        uint32_t a[4];
+        get_cpuid(0, a);
+
+        if (a[0] >= 1)
+        {
+            get_cpuid(1, a);
+            supported = ((a[2] & (1 << 9)) && (a[2] & (1 << 1)) && (a[2] & (1 << 19)));
+        }
+    }
+    return supported;
+}
+
+void crc32_init_x86(crc32_ctx* ctx);
+void crc32_update_x86(crc32_ctx* ctx, const void* buffer, size_t size);
+uint32_t crc32_done_x86(crc32_ctx* ctx);
+
+#endif
+
 static const uint32_t crc32[4][256] =
 {
     {
@@ -147,13 +196,28 @@ static const uint32_t crc32[4][256] =
 
 void crc32_init(crc32_ctx* ctx)
 {
-    ctx->crc = 0xffffffff;
+#if PLATFORM_SUPPORTS_PCLMUL
+    if (crc32_supported_x86())
+    {
+        crc32_init_x86(ctx);
+        return;
+    }
+#endif
+    ctx->crc[0] = 0xffffffff;
 }
 
 void crc32_update(crc32_ctx* ctx, const void* buffer, size_t size)
 {
+#if PLATFORM_SUPPORTS_PCLMUL
+    if (crc32_supported_x86())
+    {
+        crc32_update_x86(ctx, buffer, size);
+        return;
+    }
+#endif
+
     const uint8_t* buffer8 = buffer;
-    uint32_t value = ctx->crc;
+    uint32_t value = ctx->crc[0];
     while (size >= 4)
     {
 #if defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN
@@ -180,10 +244,17 @@ void crc32_update(crc32_ctx* ctx, const void* buffer, size_t size)
         value = (value >> 8) ^ crc32[0][(uint8_t)value ^ *buffer8++];
     }
 
-    ctx->crc = value;
+    ctx->crc[0] = value;
 }
 
 uint32_t crc32_done(crc32_ctx* ctx)
 {
-    return ~ctx->crc;
+#if PLATFORM_SUPPORTS_PCLMUL
+    if (crc32_supported_x86())
+    {
+        return crc32_done_x86(ctx);
+    }
+#endif
+
+    return ~ctx->crc[0];
 }

pkg2zip_crc32.h

@@ -3,7 +3,7 @@
 #include "pkg2zip_utils.h"
 
 typedef struct {
-    MSVC_ALIGN(16) uint32_t crc GCC_ALIGN(16);
+    uint32_t PKG_ALIGN(16) crc[4 * 5];
 } crc32_ctx;
 
 void crc32_init(crc32_ctx* ctx);

pkg2zip_crc32_x86.c

@@ -0,0 +1,336 @@
+#include "pkg2zip_crc32.h"
+
+#include <wmmintrin.h> // PCLMUL
+#include <tmmintrin.h> // SSSE3
+#include <smmintrin.h> // SSS4
+
+// Whitepaper: https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+// ZLIB licensed code from https://github.com/jtkukunas/zlib/blob/master/crc_folding.c
+
+static const uint32_t PKG_ALIGN(16) shift_table[] = {
+    0x84838281, 0x88878685, 0x8c8b8a89, 0x008f8e8d,
+    0x85848382, 0x89888786, 0x8d8c8b8a, 0x01008f8e,
+    0x86858483, 0x8a898887, 0x8e8d8c8b, 0x0201008f,
+    0x87868584, 0x8b8a8988, 0x8f8e8d8c, 0x03020100,
+    0x88878685, 0x8c8b8a89, 0x008f8e8d, 0x04030201,
+    0x89888786, 0x8d8c8b8a, 0x01008f8e, 0x05040302,
+    0x8a898887, 0x8e8d8c8b, 0x0201008f, 0x06050403,
+    0x8b8a8988, 0x8f8e8d8c, 0x03020100, 0x07060504,
+    0x8c8b8a89, 0x008f8e8d, 0x04030201, 0x08070605,
+    0x8d8c8b8a, 0x01008f8e, 0x05040302, 0x09080706,
+    0x8e8d8c8b, 0x0201008f, 0x06050403, 0x0a090807,
+    0x8f8e8d8c, 0x03020100, 0x07060504, 0x0b0a0908,
+    0x008f8e8d, 0x04030201, 0x08070605, 0x0c0b0a09,
+    0x01008f8e, 0x05040302, 0x09080706, 0x0d0c0b0a,
+    0x0201008f, 0x06050403, 0x0a090807, 0x0e0d0c0b,
+};
+
+#define FOLD1(xmm0, xmm1, xmm2, xmm3) do         \
+{                                                \
+    const __m128i fold4 = _mm_set_epi32(         \
+        0x00000001, 0x54442bd4,                  \
+        0x00000001, 0xc6e41596);                 \
+                                                 \
+    __m128i r0, r1, r2, r3, a, b;                \
+                                                 \
+    r0 = xmm1;                                   \
+    r1 = xmm2;                                   \
+    r2 = xmm3;                                   \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \
+    r3 = _mm_xor_si128(a, b);                    \
+                                                 \
+    xmm0 = r0;                                   \
+    xmm1 = r1;                                   \
+    xmm2 = r2;                                   \
+    xmm3 = r3;                                   \
+} while (0)
+
+#define FOLD2(xmm0, xmm1, xmm2, xmm3) do         \
+{                                                \
+    const __m128i fold4 = _mm_set_epi32(         \
+        0x00000001, 0x54442bd4,                  \
+        0x00000001, 0xc6e41596);                 \
+                                                 \
+    __m128i r0, r1, r2, r3, a, b;                \
+                                                 \
+    r0 = xmm2;                                   \
+    r1 = xmm3;                                   \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \
+    r2 = _mm_xor_si128(a, b);                    \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm1, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm1, fold4, 0x10); \
+    r3 = _mm_xor_si128(a, b);                    \
+                                                 \
+    xmm0 = r0;                                   \
+    xmm1 = r1;                                   \
+    xmm2 = r2;                                   \
+    xmm3 = r3;                                   \
+} while (0)
+
+#define FOLD3(xmm0, xmm1, xmm2, xmm3) do         \
+{                                                \
+    const __m128i fold4 = _mm_set_epi32(         \
+        0x00000001, 0x54442bd4,                  \
+        0x00000001, 0xc6e41596);                 \
+                                                 \
+    __m128i r0, r1, r2, r3, a, b;                \
+                                                 \
+    r0 = xmm3;                                   \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \
+    r1 = _mm_xor_si128(a, b);                    \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm1, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm1, fold4, 0x10); \
+    r2 = _mm_xor_si128(a, b);                    \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm2, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm2, fold4, 0x10); \
+    r3 = _mm_xor_si128(a, b);                    \
+                                                 \
+    xmm0 = r0;                                   \
+    xmm1 = r1;                                   \
+    xmm2 = r2;                                   \
+    xmm3 = r3;                                   \
+} while (0)
+
+#define FOLD4(xmm0, xmm1, xmm2, xmm3) do         \
+{                                                \
+    const __m128i fold4 = _mm_set_epi32(         \
+        0x00000001, 0x54442bd4,                  \
+        0x00000001, 0xc6e41596);                 \
+                                                 \
+    __m128i a, b;                                \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm0, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm0, fold4, 0x10); \
+    xmm0 = _mm_xor_si128(a, b);                  \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm1, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm1, fold4, 0x10); \
+    xmm1 = _mm_xor_si128(a, b);                  \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm2, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm2, fold4, 0x10); \
+    xmm2 = _mm_xor_si128(a, b);                  \
+                                                 \
+    a = _mm_clmulepi64_si128(xmm3, fold4, 0x01); \
+    b = _mm_clmulepi64_si128(xmm3, fold4, 0x10); \
+    xmm3 = _mm_xor_si128(a, b);                  \
+} while (0)
+
+#define PARTIAL(len, xmm0, xmm1, xmm2, xmm3, xmm4) do                 \
+{                                                                     \
+    const __m128i fold4 = _mm_set_epi32(                              \
+        0x00000001, 0x54442bd4,                                       \
+        0x00000001, 0xc6e41596);                                      \
+    const __m128i mask = _mm_set1_epi32(0x80808080);                  \
+                                                                      \
+    __m128i shl = _mm_load_si128((__m128i *)shift_table + (len - 1)); \
+    __m128i shr = _mm_xor_si128(shl, mask);                           \
+                                                                      \
+    __m128i a, b, r;                                                  \
+    __m128i tmp = _mm_shuffle_epi8(xmm0, shl);                        \
+                                                                      \
+    a = _mm_shuffle_epi8(xmm0, shr);                                  \
+    b = _mm_shuffle_epi8(xmm1, shl);                                  \
+    xmm0 = _mm_or_si128(a, b);                                        \
+                                                                      \
+    a = _mm_shuffle_epi8(xmm1, shr);                                  \
+    b = _mm_shuffle_epi8(xmm2, shl);                                  \
+    xmm1 = _mm_or_si128(a, b);                                        \
+                                                                      \
+    a = _mm_shuffle_epi8(xmm2, shr);                                  \
+    b = _mm_shuffle_epi8(xmm3, shl);                                  \
+    xmm2 = _mm_or_si128(a, b);                                        \
+                                                                      \
+    a = _mm_shuffle_epi8(xmm3, shr);                                  \
+    b = _mm_shuffle_epi8(xmm4, shl);                                  \
+    xmm4 = b;                                                         \
+    r = _mm_or_si128(a, b);                                           \
+                                                                      \
+    a = _mm_clmulepi64_si128(tmp, fold4, 0x10);                       \
+    b = _mm_clmulepi64_si128(tmp, fold4, 0x01);                       \
+                                                                      \
+    r = _mm_xor_si128(r, a);                                          \
+    r = _mm_xor_si128(r, b);                                          \
+    xmm3 = r;                                                         \
+} while(0)
+
+void crc32_init_x86(crc32_ctx* ctx)
+{
+    __m128i init = _mm_cvtsi32_si128(0x9db42487);
+    __m128i zero = _mm_setzero_si128();
+
+    _mm_store_si128((__m128i*)ctx->crc + 0, init);
+    _mm_store_si128((__m128i*)ctx->crc + 1, zero);
+    _mm_store_si128((__m128i*)ctx->crc + 2, zero);
+    _mm_store_si128((__m128i*)ctx->crc + 3, zero);
+}
+
+void crc32_update_x86(crc32_ctx* ctx, const void* buffer, size_t size)
+{
+    const uint8_t* buffer8 = buffer;
+
+    __m128i xmm0 = _mm_load_si128((__m128i*)ctx->crc + 0);
+    __m128i xmm1 = _mm_load_si128((__m128i*)ctx->crc + 1);
+    __m128i xmm2 = _mm_load_si128((__m128i*)ctx->crc + 2);
+    __m128i xmm3 = _mm_load_si128((__m128i*)ctx->crc + 3);
+    __m128i xmm4 = _mm_load_si128((__m128i*)ctx->crc + 4);
+
+    if (size < 16)
+    {
+        if (size == 0)
+        {
+            return;
+        }
+        xmm4 = _mm_loadu_si128((__m128i *)buffer8);
+        goto partial;
+    }
+
+    uint32_t prefix = (0 - (uintptr_t)buffer8) & 0xF;
+    if (prefix != 0)
+    {
+        xmm4 = _mm_loadu_si128((__m128i *)buffer8);
+        buffer8 += prefix;
+        size -= prefix;
+
+        PARTIAL(prefix, xmm0, xmm1, xmm2, xmm3, xmm4);
+    }
+
+    while (size >= 64)
+    {
+        __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0);
+        __m128i t1 = _mm_load_si128((__m128i *)buffer8 + 1);
+        __m128i t2 = _mm_load_si128((__m128i *)buffer8 + 2);
+        __m128i t3 = _mm_load_si128((__m128i *)buffer8 + 3);
+
+        FOLD4(xmm0, xmm1, xmm2, xmm3);
+
+        xmm0 = _mm_xor_si128(xmm0, t0);
+        xmm1 = _mm_xor_si128(xmm1, t1);
+        xmm2 = _mm_xor_si128(xmm2, t2);
+        xmm3 = _mm_xor_si128(xmm3, t3);
+
+        buffer8 += 64;
+        size -= 64;
+    }
+
+    if (size >= 48)
+    {
+        __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0);
+        __m128i t1 = _mm_load_si128((__m128i *)buffer8 + 1);
+        __m128i t2 = _mm_load_si128((__m128i *)buffer8 + 2);
+
+        FOLD3(xmm0, xmm1, xmm2, xmm3);
+
+        xmm1 = _mm_xor_si128(xmm1, t0);
+        xmm2 = _mm_xor_si128(xmm2, t1);
+        xmm3 = _mm_xor_si128(xmm3, t2);
+
+        buffer8 += 48;
+        size -= 48;
+    }
+    else if (size >= 32)
+    {
+        __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0);
+        __m128i t1 = _mm_load_si128((__m128i *)buffer8 + 1);
+
+        FOLD2(xmm0, xmm1, xmm2, xmm3);
+
+        xmm2 = _mm_xor_si128(xmm2, t0);
+        xmm3 = _mm_xor_si128(xmm3, t1);
+
+        buffer8 += 32;
+        size -= 32;
+    }
+    else if (size >= 16)
+    {
+        __m128i t0 = _mm_load_si128((__m128i *)buffer8 + 0);
+
+        FOLD1(xmm0, xmm1, xmm2, xmm3);
+
+        xmm3 = _mm_xor_si128(xmm3, t0);
+
+        buffer8 += 16;
+        size -= 16;
+    }
+
+    if (size == 0)
+    {
+        goto done;
+    }
+
+    xmm4 = _mm_load_si128((__m128i *)buffer8);
+
+partial:
+    PARTIAL(size, xmm0, xmm1, xmm2, xmm3, xmm4);
+
+done:
+    _mm_store_si128((__m128i*)ctx->crc + 0, xmm0);
+    _mm_store_si128((__m128i*)ctx->crc + 1, xmm1);
+    _mm_store_si128((__m128i*)ctx->crc + 2, xmm2);
+    _mm_store_si128((__m128i*)ctx->crc + 3, xmm3);
+    _mm_store_si128((__m128i*)ctx->crc + 4, xmm4);
+}
+
+uint32_t crc32_done_x86(crc32_ctx* ctx)
+{
+    const __m128i mask1 = _mm_setr_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000);
+    const __m128i mask2 = _mm_setr_epi32(0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
+
+    __m128i xmm0 = _mm_load_si128((__m128i*)ctx->crc + 0);
+    __m128i xmm1 = _mm_load_si128((__m128i*)ctx->crc + 1);
+    __m128i xmm2 = _mm_load_si128((__m128i*)ctx->crc + 2);
+    __m128i xmm3 = _mm_load_si128((__m128i*)ctx->crc + 3);
+
+    __m128i fold;
+    __m128i a, b, t;
+
+    fold = _mm_setr_epi32(0xccaa009e, 0x00000000, 0x751997d0, 0x00000001);
+
+    a = _mm_clmulepi64_si128(xmm0, fold, 0x10);
+    b = _mm_clmulepi64_si128(xmm0, fold, 0x01);
+    xmm1 = _mm_xor_si128(xmm1, a);
+    xmm1 = _mm_xor_si128(xmm1, b);
+
+    a = _mm_clmulepi64_si128(xmm1, fold, 0x10);
+    b = _mm_clmulepi64_si128(xmm1, fold, 0x01);
+    xmm2 = _mm_xor_si128(xmm2, a);
+    xmm2 = _mm_xor_si128(xmm2, b);
+
+    a = _mm_clmulepi64_si128(xmm2, fold, 0x10);
+    b = _mm_clmulepi64_si128(xmm2, fold, 0x01);
+    xmm3 = _mm_xor_si128(xmm3, a);
+    xmm3 = _mm_xor_si128(xmm3, b);
+
+    fold = _mm_setr_epi32(0xccaa009e, 0x00000000, 0x63cd6124, 0x00000001);
+
+    a = _mm_clmulepi64_si128(xmm3, fold, 0);
+    b = _mm_srli_si128(xmm3, 8);
+    a = _mm_xor_si128(a, b);
+
+    b = _mm_slli_si128(a, 4);
+    b = _mm_clmulepi64_si128(b, fold, 0x10);
+    t = _mm_xor_si128(a, b);
+    t = _mm_and_si128(t, mask2);
+
+    fold = _mm_setr_epi32(0xf7011640, 0x00000001, 0xdb710640, 0x00000001);
+
+    a = _mm_clmulepi64_si128(t, fold, 0);
+    a = _mm_xor_si128(a, t);
+    a = _mm_and_si128(a, mask1);
+
+    b = _mm_clmulepi64_si128(a, fold, 0x10);
+    b = _mm_xor_si128(b, a);
+    b = _mm_xor_si128(b, t);
+
+    uint32_t crc = _mm_extract_epi32(b, 2);
+    return ~crc;
+}

pkg2zip_utils.h

@@ -5,16 +5,10 @@
 
 #if defined(_MSC_VER)
 #  define NORETURN __declspec(noreturn)
+#  define PKG_ALIGN(x) __declspec(align(x))
 #else
 #  define NORETURN __attribute__((noreturn))
-#endif
+#  define PKG_ALIGN(x) __attribute__((aligned(x)))
-
-#if defined(_MSC_VER)
-#  define MSVC_ALIGN(x) __declspec(align(x))
-#  define GCC_ALIGN(x)
-#else
-#  define MSVC_ALIGN(x)
-#  define GCC_ALIGN(x) __attribute__((aligned(x)))
 #endif
 
 void NORETURN fatal(const char* msg, ...);