src/leveldb/port/port_posix_sse.cc

   1 // Copyright 2016 The LevelDB Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   4 //
   5 // A portable implementation of crc32c, optimized to handle
   6 // four bytes at a time.
   7 //
   8 // In a separate source file to allow this accelerated CRC32C function to be
   9 // compiled with the appropriate compiler flags to enable x86 SSE 4.2
  10 // instructions.
  11
  12 #include <stdint.h>
  13 #include <string.h>
  14 #include "port/port.h"
  15
  16 #if defined(LEVELDB_PLATFORM_POSIX_SSE)
  17
  18 #if defined(_MSC_VER)
  19 #include <intrin.h>
  20 #elif defined(__GNUC__) && defined(__SSE4_2__)
  21 #include <nmmintrin.h>
  22 #endif
  23
  24 #endif  // defined(LEVELDB_PLATFORM_POSIX_SSE)
  25
  26 namespace leveldb {
  27 namespace port {
  28
  29 #if defined(LEVELDB_PLATFORM_POSIX_SSE)
  30
  31 // Used to fetch a naturally-aligned 32-bit word in little endian byte-order
  32 static inline uint32_t LE_LOAD32(const uint8_t *p) {
  33   // SSE is x86 only, so ensured that |p| is always little-endian.
  34   uint32_t word;
  35   memcpy(&word, p, sizeof(word));
  36   return word;
  37 }
  38
  39 #if defined(_M_X64) || defined(__x86_64__)  // LE_LOAD64 is only used on x64.
  40
  41 // Used to fetch a naturally-aligned 64-bit word in little endian byte-order
  42 static inline uint64_t LE_LOAD64(const uint8_t *p) {
  43   uint64_t dword;
  44   memcpy(&dword, p, sizeof(dword));
  45   return dword;
  46 }
  47
  48 #endif  // defined(_M_X64) || defined(__x86_64__)
  49
  50 #endif  // defined(LEVELDB_PLATFORM_POSIX_SSE)
  51
  52 // For further improvements see Intel publication at:
  53 // http://download.intel.com/design/intarch/papers/323405.pdf
  54 uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) {
  55 #if !defined(LEVELDB_PLATFORM_POSIX_SSE)
  56   return 0;
  57 #else
  58
  59   const uint8_t *p = reinterpret_cast<const uint8_t *>(buf);
  60   const uint8_t *e = p + size;
  61   uint32_t l = crc ^ 0xffffffffu;
  62
  63 #define STEP1 do {                              \
  64     l = _mm_crc32_u8(l, *p++);                  \
  65 } while (0)
  66 #define STEP4 do {                              \
  67     l = _mm_crc32_u32(l, LE_LOAD32(p));         \
  68     p += 4;                                     \
  69 } while (0)
  70 #define STEP8 do {                              \
  71     l = _mm_crc32_u64(l, LE_LOAD64(p));         \
  72     p += 8;                                     \
  73 } while (0)
  74
  75   if (size > 16) {
  76     // Process unaligned bytes
  77     for (unsigned int i = reinterpret_cast<uintptr_t>(p) % 8; i; --i) {
  78       STEP1;
  79     }
  80
  81     // _mm_crc32_u64 is only available on x64.
  82 #if defined(_M_X64) || defined(__x86_64__)
  83     // Process 8 bytes at a time
  84     while ((e-p) >= 8) {
  85       STEP8;
  86     }
  87     // Process 4 bytes at a time
  88     if ((e-p) >= 4) {
  89       STEP4;
  90     }
  91 #else  // !(defined(_M_X64) || defined(__x86_64__))
  92     // Process 4 bytes at a time
  93     while ((e-p) >= 4) {
  94       STEP4;
  95     }
  96 #endif  // defined(_M_X64) || defined(__x86_64__)
  97   }
  98   // Process the last few bytes
  99   while (p != e) {
 100     STEP1;
 101   }
 102 #undef STEP8
 103 #undef STEP4
 104 #undef STEP1
 105   return l ^ 0xffffffffu;
 106 #endif  // defined(LEVELDB_PLATFORM_POSIX_SSE)
 107 }
 108
 109 }  // namespace port
 110 }  // namespace leveldb