From b9ec2aa3454d46e4a06293d6fcdcc0a6be7d57e0 Mon Sep 17 00:00:00 2001 From: yonik Date: Wed, 13 May 2015 13:35:50 -0400 Subject: [PATCH 1/4] implement 128 bit murmurhash, test with more variety of seeds --- src/util/hash/MurmurHash3.java | 108 ++++++++++++++++++++++++++-- test/util/hash/TestMurmurHash3.java | 35 +++++++-- 2 files changed, 132 insertions(+), 11 deletions(-) diff --git a/src/util/hash/MurmurHash3.java b/src/util/hash/MurmurHash3.java index 4e95e77..48ccf9c 100644 --- a/src/util/hash/MurmurHash3.java +++ b/src/util/hash/MurmurHash3.java @@ -1,22 +1,60 @@ package util.hash; /** - * The MurmurHash3 algorithm was created by Austin Appleby. This java port was authored by - * Yonik Seeley and is placed into the public domain. The author hereby disclaims copyright - * to this source code. + * The MurmurHash3 algorithm was created by Austin Appleby and placed in the public domain. + * This java port was authored by Yonik Seeley and also placed into the public domain. + * The author hereby disclaims copyright to this source code. *

* This produces exactly the same hash values as the final C++ * version of MurmurHash3 and is thus suitable for producing the same hash values across * platforms. *

* The 32 bit x86 version of this hash should be the fastest variant for relatively short keys like ids. + * murmurhash3_x64_128 is a good choice for longer strings or if you need more than 32 bits of hash. *

* Note - The x86 and x64 versions do _not_ produce the same results, as the * algorithms are optimized for their respective platforms. *

* See http://github.com/yonik/java_util for future updates to this file. */ -public class MurmurHash3 { +public final class MurmurHash3 { + + /** 128 bits of state */ + public static final class LongPair { + long val1; + long val2; + } + + public static final int fmix32(int h) { + h ^= h >>> 16; + h *= 0x85ebca6b; + h ^= h >>> 13; + h *= 0xc2b2ae35; + h ^= h >>> 16; + return h; + } + + public static final long fmix64(long k) { + k ^= k >>> 33; + k *= 0xff51afd7ed558ccdL; + k ^= k >>> 33; + k *= 0xc4ceb9fe1a85ec53L; + k ^= k >>> 33; + return k; + } + + /** Gets a long from a byte buffer in little endian byte order. */ + public static final long getLongLittleEndian(byte[] buf, int offset) { + return ((long)buf[offset+7] << 56) // no mask needed + | ((buf[offset+6] & 0xffL) << 48) + | ((buf[offset+5] & 0xffL) << 40) + | ((buf[offset+4] & 0xffL) << 32) + | ((buf[offset+3] & 0xffL) << 24) + | ((buf[offset+2] & 0xffL) << 16) + | ((buf[offset+1] & 0xffL) << 8) + | ((buf[offset ] & 0xffL)); // no shift needed + } + /** Returns the MurmurHash3_x86_32 hash. */ public static int murmurhash3_x86_32(byte[] data, int offset, int len, int seed) { @@ -196,4 +234,66 @@ else if (code < 0xD800 || code > 0xDFFF || pos>=end) { return h1; } + + /** Returns the MurmurHash3_x64_128 hash, placing the result in "out". */ + public static void murmurhash3_x64_128(byte[] key, int offset, int len, int seed, LongPair out) { + // The original algorithm does have a 32 bit unsigned seed. + // We have to mask to match the behavior of the unsigned types and prevent sign extension. + long h1 = seed & 0x00000000FFFFFFFFL; + long h2 = seed & 0x00000000FFFFFFFFL; + + final long c1 = 0x87c37b91114253d5L; + final long c2 = 0x4cf5ad432745937fL; + + int roundedEnd = offset + (len & 0xFFFFFFF0); // round down to 16 byte block + for (int i=offset; i #include "MurmurHash3.h" using namespace std; @@ -120,11 +128,24 @@ int main(int argc, char** argv) { // to catch errors like signed vs unsigned shifting, etc. val[i] = (char)hash; } + uint32_t seed = 1; for (int len=0; len Date: Thu, 14 May 2015 16:20:23 -0400 Subject: [PATCH 2/4] make LongPair members public --- src/util/hash/MurmurHash3.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/hash/MurmurHash3.java b/src/util/hash/MurmurHash3.java index 48ccf9c..5d5d18e 100644 --- a/src/util/hash/MurmurHash3.java +++ b/src/util/hash/MurmurHash3.java @@ -21,8 +21,8 @@ public final class MurmurHash3 { /** 128 bits of state */ public static final class LongPair { - long val1; - long val2; + public long val1; + public long val2; } public static final int fmix32(int h) { From a239ff541d1d267c208c420ddbbf940934379c0c Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Thu, 17 May 2018 20:53:34 -0700 Subject: [PATCH 3/4] Suppress fallthrough warnings Allows compilation when -Werror specified. --- src/util/hash/MurmurHash3.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/hash/MurmurHash3.java b/src/util/hash/MurmurHash3.java index 5d5d18e..d0a9e43 100644 --- a/src/util/hash/MurmurHash3.java +++ b/src/util/hash/MurmurHash3.java @@ -57,6 +57,7 @@ public static final long getLongLittleEndian(byte[] buf, int offset) { /** Returns the MurmurHash3_x86_32 hash. */ + @SuppressWarnings("fallthrough") public static int murmurhash3_x86_32(byte[] data, int offset, int len, int seed) { final int c1 = 0xcc9e2d51; @@ -236,6 +237,7 @@ else if (code < 0xD800 || code > 0xDFFF || pos>=end) { /** Returns the MurmurHash3_x64_128 hash, placing the result in "out". */ + @SuppressWarnings("fallthrough") public static void murmurhash3_x64_128(byte[] key, int offset, int len, int seed, LongPair out) { // The original algorithm does have a 32 bit unsigned seed. // We have to mask to match the behavior of the unsigned types and prevent sign extension. From 435ae306d2f2c077d981ab4de5c9ac3c45f92a4b Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Tue, 30 Jul 2019 18:54:03 -0400 Subject: [PATCH 4/4] Update README.txt --- README.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.txt b/README.txt index 2aeba02..5e56833 100644 --- a/README.txt +++ b/README.txt @@ -6,5 +6,7 @@ As such, unless otherwise noted, this code has been authored by Yonik Seeley and placed into the public domain to allow people to use or license as they see fit. -util/hash/MurmurHash3: a fast, high quality hash function. +util/hash/MurmurHash3: + A fast, high quality hash function. This version can also calculate + the hash of the UTF-8 encoding of a String without converting to a UTF-8 byte[].