37 #include <boost/endian/buffers.hpp>
39 #if defined(__SSE4_2__) && defined(__x86_64__)
40 #include <nmmintrin.h>
41 #define _mm_crc32_u64_impl _mm_crc32_u64
54 const uint64_t kMul = 0x9ddfea08eb382d69ULL;
66 #define bswap_32(x) _byteswap_ulong(x)
67 #define bswap_64(x) _byteswap_uint64(x)
69 #elif defined(__APPLE__)
72 #include <libkern/OSByteOrder.h>
73 #define bswap_32(x) OSSwapInt32(x)
74 #define bswap_64(x) OSSwapInt64(x)
76 #elif defined(__sun) || defined(sun)
78 #include <sys/byteorder.h>
79 #define bswap_32(x) BSWAP_32(x)
80 #define bswap_64(x) BSWAP_64(x)
82 #elif defined(__FreeBSD__)
84 #include <sys/endian.h>
85 #define bswap_32(x) bswap32(x)
86 #define bswap_64(x) bswap64(x)
88 #elif defined(__OpenBSD__)
90 #include <sys/types.h>
91 #define bswap_32(x) swap32(x)
92 #define bswap_64(x) swap64(x)
94 #elif defined(__NetBSD__)
96 #include <sys/types.h>
97 #include <machine/bswap.h>
98 #if defined(__BSWAP_RENAME) && !defined(__bswap_32)
99 #define bswap_32(x) bswap32(x)
100 #define bswap_64(x) bswap64(x)
106 #include <byteswap.h>
112 #if HAVE_BUILTIN_EXPECT
113 #define LIKELY(x) (__builtin_expect(!!(x), 1))
115 #define LIKELY(x) (x)
119 static uint64_t Fetch64(
const char *p) {
120 boost::endian::little_uint64_buf_t result;
121 memcpy(&result, p,
sizeof(result));
122 return result.value();
125 static uint32_t Fetch32(
const char *p) {
126 boost::endian::little_uint32_buf_t result;
127 memcpy(&result, p,
sizeof(result));
128 return result.value();
132 static const uint64_t k0 = 0xc3a5c85c97cb3127ULL;
133 static const uint64_t k1 = 0xb492b66fbe98f273ULL;
134 static const uint64_t k2 = 0x9ae16a3b2f90404fULL;
137 static const uint32_t c1 = 0xcc9e2d51;
138 static const uint32_t c2 = 0x1b873593;
141 static uint32_t fmix(uint32_t h)
151 static uint32_t Rotate32(uint32_t val,
int shift) {
153 return shift == 0 ? val : ((val >> shift) | (val << (32 - shift)));
157 #define PERMUTE3(a, b, c) do { std::swap(a, b); std::swap(a, c); } while (0)
159 static uint32_t Mur(uint32_t a, uint32_t h) {
166 return h * 5 + 0xe6546b64;
169 static uint32_t Hash32Len13to24(
const char *s,
size_t len) {
170 uint32_t a = Fetch32(s - 4 + (len >> 1));
171 uint32_t b = Fetch32(s + 4);
172 uint32_t c = Fetch32(s + len - 8);
173 uint32_t d = Fetch32(s + (len >> 1));
174 uint32_t e = Fetch32(s);
175 uint32_t f = Fetch32(s + len - 4);
178 return fmix(Mur(f, Mur(e, Mur(d, Mur(c, Mur(b, Mur(a, h)))))));
181 static uint32_t Hash32Len0to4(
const char *s,
size_t len) {
184 for (
size_t i = 0; i < len; i++) {
185 signed char v = s[i];
189 return fmix(Mur(b, Mur(len, c)));
192 static uint32_t Hash32Len5to12(
const char *s,
size_t len) {
193 uint32_t a = len, b = len * 5, c = 9, d = b;
195 b += Fetch32(s + len - 4);
196 c += Fetch32(s + ((len >> 1) & 4));
197 return fmix(Mur(c, Mur(b, Mur(a, d))));
203 (len <= 4 ? Hash32Len0to4(s, len) : Hash32Len5to12(s, len)) :
204 Hash32Len13to24(s, len);
208 uint32_t h = len, g = c1 * len, f = g;
209 uint32_t a0 = Rotate32(Fetch32(s + len - 4) * c1, 17) * c2;
210 uint32_t a1 = Rotate32(Fetch32(s + len - 8) * c1, 17) * c2;
211 uint32_t a2 = Rotate32(Fetch32(s + len - 16) * c1, 17) * c2;
212 uint32_t a3 = Rotate32(Fetch32(s + len - 12) * c1, 17) * c2;
213 uint32_t a4 = Rotate32(Fetch32(s + len - 20) * c1, 17) * c2;
216 h = h * 5 + 0xe6546b64;
219 h = h * 5 + 0xe6546b64;
222 g = g * 5 + 0xe6546b64;
225 g = g * 5 + 0xe6546b64;
228 f = f * 5 + 0xe6546b64;
229 size_t iters = (len - 1) / 20;
231 uint32_t a0 = Rotate32(Fetch32(s) * c1, 17) * c2;
232 uint32_t a1 = Fetch32(s + 4);
233 uint32_t a2 = Rotate32(Fetch32(s + 8) * c1, 17) * c2;
234 uint32_t a3 = Rotate32(Fetch32(s + 12) * c1, 17) * c2;
235 uint32_t a4 = Fetch32(s + 16);
238 h = h * 5 + 0xe6546b64;
244 g = g * 5 + 0xe6546b64;
247 h = h * 5 + 0xe6546b64;
255 }
while (--iters != 0);
256 g = Rotate32(g, 11) * c1;
257 g = Rotate32(g, 17) * c1;
258 f = Rotate32(f, 11) * c1;
259 f = Rotate32(f, 17) * c1;
260 h = Rotate32(h + g, 19);
261 h = h * 5 + 0xe6546b64;
262 h = Rotate32(h, 17) * c1;
263 h = Rotate32(h + f, 19);
264 h = h * 5 + 0xe6546b64;
265 h = Rotate32(h, 17) * c1;
271 static uint64_t Rotate(uint64_t val,
int shift) {
273 return shift == 0 ? val : ((val >> shift) | (val << (64 - shift)));
276 static uint64_t ShiftMix(uint64_t val) {
277 return val ^ (val >> 47);
280 static uint64_t HashLen16(uint64_t u, uint64_t v) {
284 static uint64_t HashLen16(uint64_t u, uint64_t v, uint64_t mul) {
286 uint64_t a = (u ^ v) * mul;
288 uint64_t b = (v ^ a) * mul;
294 static uint64_t HashLen0to16(
const char *s,
size_t len) {
296 uint64_t mul = k2 + len * 2;
297 uint64_t a = Fetch64(s) + k2;
298 uint64_t b = Fetch64(s + len - 8);
299 uint64_t c = Rotate(b, 37) * mul + a;
300 uint64_t d = (Rotate(a, 25) + b) * mul;
301 return HashLen16(c, d, mul);
304 uint64_t mul = k2 + len * 2;
305 uint64_t a = Fetch32(s);
306 return HashLen16(len + (a << 3), Fetch32(s + len - 4), mul);
310 uint8_t b = s[len >> 1];
311 uint8_t c = s[len - 1];
312 uint32_t y =
static_cast<uint32_t
>(a) + (
static_cast<uint32_t
>(b) << 8);
313 uint32_t z = len + (
static_cast<uint32_t
>(c) << 2);
314 return ShiftMix(y * k2 ^ z * k0) * k2;
321 static uint64_t HashLen17to32(
const char *s,
size_t len) {
322 uint64_t mul = k2 + len * 2;
323 uint64_t a = Fetch64(s) * k1;
324 uint64_t b = Fetch64(s + 8);
325 uint64_t c = Fetch64(s + len - 8) * mul;
326 uint64_t d = Fetch64(s + len - 16) * k2;
327 return HashLen16(Rotate(a + b, 43) + Rotate(c, 30) + d,
328 a + Rotate(b + k2, 18) + c, mul);
333 static pair<uint64_t, uint64_t> WeakHashLen32WithSeeds(
334 uint64_t w, uint64_t x, uint64_t y, uint64_t z, uint64_t a, uint64_t b) {
336 b = Rotate(b + a + z, 21);
341 return make_pair(a + z, b + c);
345 static pair<uint64_t, uint64_t> WeakHashLen32WithSeeds(
346 const char* s, uint64_t a, uint64_t b) {
347 return WeakHashLen32WithSeeds(Fetch64(s),
356 static uint64_t HashLen33to64(
const char *s,
size_t len) {
357 uint64_t mul = k2 + len * 2;
358 uint64_t a = Fetch64(s) * k2;
359 uint64_t b = Fetch64(s + 8);
360 uint64_t c = Fetch64(s + len - 24);
361 uint64_t d = Fetch64(s + len - 32);
362 uint64_t e = Fetch64(s + 16) * k2;
363 uint64_t f = Fetch64(s + 24) * 9;
364 uint64_t g = Fetch64(s + len - 8);
365 uint64_t h = Fetch64(s + len - 16) * mul;
366 uint64_t u = Rotate(a + g, 43) + (Rotate(b, 30) + c) * 9;
367 uint64_t v = ((a + g) ^ d) + f + 1;
368 uint64_t w = bswap_64((u + v) * mul) + h;
369 uint64_t x = Rotate(e + f, 42) + c;
370 uint64_t y = (bswap_64((v + w) * mul) + g) * mul;
371 uint64_t z = e + f + c;
372 a = bswap_64((x + z) * mul + y) + b;
373 b = ShiftMix((z + a) * mul + d + h) * mul;
380 return HashLen0to16(s, len);
382 return HashLen17to32(s, len);
384 }
else if (len <= 64) {
385 return HashLen33to64(s, len);
390 uint64_t x = Fetch64(s + len - 40);
391 uint64_t y = Fetch64(s + len - 16) + Fetch64(s + len - 56);
392 uint64_t z = HashLen16(Fetch64(s + len - 48) + len, Fetch64(s + len - 24));
393 pair<uint64_t, uint64_t> v = WeakHashLen32WithSeeds(s + len - 64, len, z);
394 pair<uint64_t, uint64_t> w = WeakHashLen32WithSeeds(s + len - 32, y + k1, x);
395 x = x * k1 + Fetch64(s);
398 len = (len - 1) & ~
static_cast<size_t>(63);
400 x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
401 y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
403 y += v.first + Fetch64(s + 40);
404 z = Rotate(z + w.first, 33) * k1;
405 v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
406 w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
411 return HashLen16(HashLen16(v.first, w.first) + ShiftMix(y) * k1 + z,
412 HashLen16(v.second, w.second) + x);
416 uint64_t seed0, uint64_t seed1) {
417 return HashLen16(
city_hash64(s, len) - seed0, seed1);
426 uint128_t
CityMurmur(
const char *s,
size_t len, uint128_t seed) {
431 signed long l = len - 16;
433 a = ShiftMix(a * k1) * k1;
434 c = b * k1 + HashLen0to16(s, len);
435 d = ShiftMix(a + (len >= 8 ? Fetch64(s) : c));
437 c = HashLen16(Fetch64(s + len - 8) + k1, a);
438 d = HashLen16(b + len, c + Fetch64(s + len - 16));
441 a ^= ShiftMix(Fetch64(s) * k1) * k1;
444 c ^= ShiftMix(Fetch64(s + 8) * k1) * k1;
453 return uint128( a ^ b, HashLen16(b, a) );
463 pair<uint64_t, uint64_t> v, w;
466 uint64_t z = len * k1;
467 v.first = Rotate(y ^ k1, 49) * k1 + Fetch64(s);
468 v.second = Rotate(v.first, 42) * k1 + Fetch64(s + 8);
469 w.first = Rotate(y + z, 35) * k1 + x;
470 w.second = Rotate(x + Fetch64(s + 88), 53) * k1;
474 x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
475 y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
477 y += v.first + Fetch64(s + 40);
478 z = Rotate(z + w.first, 33) * k1;
479 v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
480 w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
483 x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
484 y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
486 y += v.first + Fetch64(s + 40);
487 z = Rotate(z + w.first, 33) * k1;
488 v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
489 w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
493 }
while (
LIKELY(len >= 128));
494 x += Rotate(v.first + z, 49) * k0;
495 y = y * k0 + Rotate(w.second, 37);
496 z = z * k0 + Rotate(w.first, 27);
500 for (
size_t tail_done = 0; tail_done < len; ) {
502 y = Rotate(x + y, 42) * k0 + v.second;
503 w.first += Fetch64(s + len - tail_done + 16);
504 x = x * k0 + w.first;
505 z += w.second + Fetch64(s + len - tail_done);
507 v = WeakHashLen32WithSeeds(s + len - tail_done, v.first + z, v.second);
513 x = HashLen16(x, v.first);
514 y = HashLen16(y + z, w.first);
515 return uint128( HashLen16(x + v.second, w.second) + y, HashLen16(x + w.second, y + v.second) );
529 static void CityHashCrc256Long(
const char *s,
size_t len,
530 uint32_t seed, uint64_t *result) {
531 uint64_t a = Fetch64(s + 56) + k0;
532 uint64_t b = Fetch64(s + 96) + k0;
533 uint64_t c = result[0] = HashLen16(b, len);
534 uint64_t d = result[1] = Fetch64(s + 120) * k0 + len;
535 uint64_t e = Fetch64(s + 184) + seed;
544 size_t iters = len / 240;
551 c += Fetch64(s + 8); \
552 d += Fetch64(s + 16); \
553 e += Fetch64(s + 24); \
554 f += Fetch64(s + 32); \
562 z = _mm_crc32_u64_impl(z, b + g); \
563 y = _mm_crc32_u64_impl(y, e + h); \
564 x = _mm_crc32_u64_impl(x, f + a); \
575 }
while (--iters > 0);
597 a = HashLen16(a, g + z);
600 c = HashLen16(c, z) + h;
601 d = HashLen16(d, e + result[0]);
603 h += HashLen16(x, f);
604 e = HashLen16(a, d) + g;
605 z = HashLen16(b, c) + a;
606 y = HashLen16(g, h) + c;
607 result[0] = e + z + y + x;
608 a = ShiftMix((a + y) * k0) * k0 + b;
609 result[1] += a + result[0];
610 a = ShiftMix(a * k0) * k0 + c;
611 result[2] = a + result[1];
612 a = ShiftMix((a + e) * k0) * k0;
613 result[3] = a + result[2];
617 static void CityHashCrc256Short(
const char *s,
size_t len, uint64_t *result) {
620 memset(buf + len, 0, 240 - len);
621 CityHashCrc256Long(buf, 240, ~
static_cast<uint32_t
>(len), result);
626 CityHashCrc256Long(s, len, 0, result);
628 CityHashCrc256Short(s, len, result);
634 array<uint64_t,4> buf;
647 return uint128( HashLen16(u, v + result[2]), HashLen16(Rotate(v, 32), u * k0 + result[3]) );
657 return uint128( result[2], result[3] );