From f153199bbb92df8fb0081f8fb2ff15a7c7bff5fa Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 28 Aug 2025 22:09:34 -0400 Subject: [PATCH 01/38] Update Python link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b3ad6a..ded2e28 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ When using Maven: * [Erlang](https://github.com/mpope9/exor_filter) * [Go](https://github.com/FastFilter/xorfilter) * [Java](https://github.com/komiya-atsushi/xor-filter) -* [Python](https://github.com/GreyDireWolf/pyxorfilter) +* [Python](https://github.com/FastFilter/pyfusefilter) * Rust: [1](https://github.com/bnclabs/xorfilter), [2](https://github.com/codri/xorfilter-rs), [3](https://github.com/Polochon-street/rustxorfilter) * [C#](https://github.com/jonmat/FastIndex) From be4c019193db5452bc7d6bd48c060b39a52fc4e8 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 28 Aug 2025 22:10:08 -0400 Subject: [PATCH 02/38] Update README.md From 8cc6c9960f016b995ea483043083d3f595e93d4a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Fri, 2 Jan 2026 11:14:47 -0500 Subject: [PATCH 03/38] adding fuse16 --- .../java/org/fastfilter/TestFilterType.java | 7 ++++++ .../org/fastfilter/xor/ProbabilityCFuse.java | 2 -- .../org/fastfilter/xor/ProbabilityCFuse2.java | 16 -------------- .../java/org/fastfilter/xor/SmallSetTest.java | 22 +++++++++++++++++++ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java index 00fb1bb..eb91303 100644 --- a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java +++ b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java @@ -14,6 +14,7 @@ import org.fastfilter.xor.Xor16; import org.fastfilter.xor.Xor8; import org.fastfilter.xor.XorBinaryFuse8; +import org.fastfilter.xor.XorBinaryFuse16; import org.fastfilter.xor.XorSimple; import org.fastfilter.xor.XorSimple2; import org.fastfilter.xorplus.XorPlus8; @@ -100,6 +101,12 @@ public Filter construct(long[] keys, int setting) { return XorBinaryFuse8.construct(keys); } }, + XOR_BINARY_FUSE_16 { + @Override + public Filter construct(long[] keys, int setting) { + return XorBinaryFuse16.construct(keys); + } + }, CUCKOO_8 { @Override public Filter construct(long[] keys, int setting) { diff --git a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java index 3039baf..20a9ab6 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java @@ -24,7 +24,6 @@ public class ProbabilityCFuse { public static void main(String... args) { for(int size = 1; size < 1_000_000; size *= 10) { - // for(int size = 1; size < 1_000_000; size = (size < 100) ? (size + 1) : (int) (size * 1.1)) { Data best = null; for (int segmentLengthBits = 3; segmentLengthBits < 14; segmentLengthBits++) { int segmentLength = 1 << segmentLengthBits; @@ -65,7 +64,6 @@ static Data getProbability(int size, int segmentLengthBits, double load, Data be if (best != null && d.bitsPerKey > best.bitsPerKey) { return null; } - // System.out.println(" test " + d); int successCount = 0; int testCount = Math.max(10, 10_000_000 / size); for(int seed = 0; seed < testCount; seed++) { diff --git a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java index 8a0bbe9..271414d 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java @@ -37,12 +37,6 @@ private static void testProb() { long seed = 0; long key = i; long hash = Hash.hash64(key, seed); - -// int r0 = (int) Hash.hash64(hash, 1); -// int x = Hash.reduce(r0, segmentCount); -// int h0 = x + (int) (Hash.hash64(hash, 2) & (segmentLength - 1)); -// int h1 = x + (int) (Hash.hash64(hash, 3) & (segmentLength - 1)); -// int h2 = x + (int) (Hash.hash64(hash, 4) & (segmentLength - 1)); // int r0 = (int) Hash.hash64(hash, 1); int x = Hash.reduce(r0, segmentCount * 2 + segmentLength - 1); @@ -63,16 +57,9 @@ private static void testProb() { } public static void main(String... args) { -//testProb(); -//if(true)return; - - -// for(int size = 100_000; size < 1_000_000; size *= 10) { for(int size = 1; size < 1_000_000; size *= 10) { - // for(int size = 1; size < 1_000_000; size = (size < 100) ? (size + 1) : (int) (size * 1.1)) { Data best = null; for (int segmentLengthBits = 3; segmentLengthBits <= 12; segmentLengthBits++) { -// for (int segmentLengthBits = 3; segmentLengthBits < 14; segmentLengthBits++) { int segmentLength = 1 << segmentLengthBits; if (segmentLength > size) { break; @@ -89,9 +76,6 @@ public static void main(String... args) { } if (best != null) { System.out.println(best); -// for(int i=0; i<100; i++) { -// System.out.println(i + ": " + best.data[i]); -// } } } } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java index 4899c4d..4864316 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java @@ -13,6 +13,7 @@ public void small() { Xor8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); Xor16.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); XorBinaryFuse8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); + XorBinaryFuse16.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); XorSimple.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); XorSimple2.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); } @@ -41,6 +42,19 @@ public void smallSizes() { } } + @Test + public void smallSizes16() { + long lastTime = System.currentTimeMillis(); + for (int n = 1; n < 1_500_000; n = (int) ((n * 1.01) + 7)) { + XorBinaryFuse16 f = testWithSize16(n); + long now = System.currentTimeMillis(); + if (now - lastTime > 5000) { + lastTime = now; + System.out.println("n=" + n + " " + f.toString()); + } + } + } + private static XorBinaryFuse8 testWithSize(int n) { long[] keys = new long[n]; for (int i = 0; i < n; i++) { @@ -49,4 +63,12 @@ private static XorBinaryFuse8 testWithSize(int n) { return XorBinaryFuse8.construct(keys); } + private static XorBinaryFuse16 testWithSize16(int n) { + long[] keys = new long[n]; + for (int i = 0; i < n; i++) { + keys[i] = i; + } + return XorBinaryFuse16.construct(keys); + } + } From 48bbddf9c4b7000bf979e8ab37a42f424b087f95 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Fri, 2 Jan 2026 11:19:00 -0500 Subject: [PATCH 04/38] removing xorsimple --- .../main/java/org/fastfilter/FilterType.java | 12 - .../org/fastfilter/xor/XorBinaryFuse16.java | 264 ++++++++++++++++++ .../java/org/fastfilter/xor/XorSimple.java | 95 ------- .../java/org/fastfilter/xor/XorSimple2.java | 68 ----- .../java/org/fastfilter/RegressionTests.java | 1 - .../java/org/fastfilter/TestFilterType.java | 20 -- .../java/org/fastfilter/xor/SmallSetTest.java | 2 - .../org/fastfilter/ConstructionState.java | 2 - 8 files changed, 264 insertions(+), 200 deletions(-) create mode 100644 fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java delete mode 100644 fastfilter/src/main/java/org/fastfilter/xor/XorSimple.java delete mode 100644 fastfilter/src/main/java/org/fastfilter/xor/XorSimple2.java diff --git a/fastfilter/src/main/java/org/fastfilter/FilterType.java b/fastfilter/src/main/java/org/fastfilter/FilterType.java index 4a39eb4..9076962 100644 --- a/fastfilter/src/main/java/org/fastfilter/FilterType.java +++ b/fastfilter/src/main/java/org/fastfilter/FilterType.java @@ -57,18 +57,6 @@ public Filter construct(long[] keys, int setting) { return SuccinctCountingBlockedBloomRanked.construct(keys, setting); } }, - XOR_SIMPLE { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple.construct(keys); - } - }, - XOR_SIMPLE_2 { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple2.construct(keys); - } - }, XOR_8 { @Override public Filter construct(long[] keys, int setting) { diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java new file mode 100644 index 0000000..aad3c77 --- /dev/null +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -0,0 +1,264 @@ +package org.fastfilter.xor; + +import java.util.Arrays; + +import org.fastfilter.Filter; +import org.fastfilter.utils.Hash; + +/** + * The xor binary fuse filter, a new algorithm that can replace a Bloom filter. + */ +public class XorBinaryFuse16 implements Filter { + + private static final int ARITY = 3; + + private final int segmentCount; + private final int segmentCountLength; + private final int segmentLength; + private final int segmentLengthMask; + private final int arrayLength; + private final short[] fingerprints; + private long seed; + + public XorBinaryFuse16(int segmentCount, int segmentLength) { + if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) { + throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength); + } + if (segmentCount <= 0) { + throw new IllegalArgumentException("Illegal segment count: " + segmentCount); + } + this.segmentLength = segmentLength; + this.segmentCount = segmentCount; + this.segmentLengthMask = segmentLength - 1; + this.segmentCountLength = segmentCount * segmentLength; + this.arrayLength = (segmentCount + ARITY - 1) * segmentLength; + this.fingerprints = new short[arrayLength]; + } + + public long getBitCount() { + return arrayLength * 16L; + } + + static int calculateSegmentLength(int arity, int size) { + int segmentLength; + if (arity == 3) { + segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(3.33) + 2.11); + } else if (arity == 4) { + segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(2.91) - 0.5); + } else { + // not supported + segmentLength = 65536; + } + return segmentLength; + } + + static double calculateSizeFactor(int arity, int size) { + double sizeFactor; + if (arity == 3) { + sizeFactor = Math.max(1.125, 0.875 + 0.25 * Math.log(1000000) / Math.log(size)); + } else if (arity == 4) { + sizeFactor = Math.max(1.075, 0.77 + 0.305 * Math.log(600000) / Math.log(size)); + } else { + // not supported + sizeFactor = 2.0; + } + return sizeFactor; + } + + private static int mod3(int x) { + if (x > 2) { + x -= 3; + } + return x; + } + + public static XorBinaryFuse16 construct(long[] keys) { + int size = keys.length; + int segmentLength = calculateSegmentLength(ARITY, size); + // the current implementation hardcodes a 18-bit limit to + // to the segment length. + if (segmentLength > (1 << 18)) { + segmentLength = (1 << 18); + } + double sizeFactor = calculateSizeFactor(ARITY, size); + int capacity = (int) (size * sizeFactor); + int segmentCount = (capacity + segmentLength - 1) / segmentLength - (ARITY - 1); + int arrayLength = (segmentCount + ARITY - 1) * segmentLength; + segmentCount = (arrayLength + segmentLength - 1) / segmentLength; + segmentCount = segmentCount <= ARITY - 1 ? 1 : segmentCount - (ARITY - 1); + XorBinaryFuse16 filter = new XorBinaryFuse16(segmentCount, segmentLength); + filter.addAll(keys); + return filter; + } + + private void addAll(long[] keys) { + int size = keys.length; + long[] reverseOrder = new long[size + 1]; + byte[] reverseH = new byte[size]; + int reverseOrderPos = 0; + + // the lowest 2 bits are the h index (0, 1, or 2) + // so we only have 6 bits for counting; + // but that's sufficient + byte[] t2count = new byte[arrayLength]; + long[] t2hash = new long[arrayLength]; + int[] alone = new int[arrayLength]; + int hashIndex = 0; + // the array h0, h1, h2, h0, h1, h2 + int[] h012 = new int[5]; + int blockBits = 1; + while ((1 << blockBits) < segmentCount) { + blockBits++; + } + int block = 1 << blockBits; + mainloop: + while (true) { + reverseOrder[size] = 1; + int[] startPos = new int[block]; + for (int i = 0; i < 1 << blockBits; i++) { + startPos[i] = (int) ((long) i * size / block); + } + // counting sort + + for (long key : keys) { + long hash = Hash.hash64(key, seed); + int segmentIndex = (int) (hash >>> (64 - blockBits)); + // We only overwrite when the hash was zero. Zero hash values + // may be misplaced (unlikely). + while (reverseOrder[startPos[segmentIndex]] != 0) { + segmentIndex++; + segmentIndex &= (1 << blockBits) - 1; + } + reverseOrder[startPos[segmentIndex]] = hash; + startPos[segmentIndex]++; + } + byte countMask = 0; + for (int i = 0; i < size; i++) { + long hash = reverseOrder[i]; + for (int hi = 0; hi < 3; hi++) { + int index = getHashFromHash(hash, hi); + t2count[index] += 4; + t2count[index] ^= hi; + t2hash[index] ^= hash; + countMask |= t2count[index]; + } + } + startPos = null; + if (countMask < 0) { + // we have a possible counter overflow + continue mainloop; + } + + reverseOrderPos = 0; + int alonePos = 0; + for (int i = 0; i < arrayLength; i++) { + alone[alonePos] = i; + int inc = (t2count[i] >> 2) == 1 ? 1 : 0; + alonePos += inc; + } + + while (alonePos > 0) { + alonePos--; + int index = alone[alonePos]; + if ((t2count[index] >> 2) == 1) { + // It is still there! + long hash = t2hash[index]; + byte found = (byte) (t2count[index] & 3); + + reverseH[reverseOrderPos] = found; + reverseOrder[reverseOrderPos] = hash; + + h012[0] = getHashFromHash(hash, 0); + h012[1] = getHashFromHash(hash, 1); + h012[2] = getHashFromHash(hash, 2); + + int index3 = h012[mod3(found + 1)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 1); + t2hash[index3] ^= hash; + + index3 = h012[mod3(found + 2)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 2); + t2hash[index3] ^= hash; + + reverseOrderPos++; + } + } + + if (reverseOrderPos == size) { + break; + } + hashIndex++; + Arrays.fill(t2count, (byte) 0); + Arrays.fill(t2hash, 0); + Arrays.fill(reverseOrder, 0); + + if (hashIndex > 100) { + // if construction doesn't succeed eventually, + // then there is likely a problem with the hash function + // let us not crash the system: + for(int i = 0; i < fingerprints.length; i++) { + fingerprints[i] = (short)0xFFFF; + } + return; + } + // use a new random numbers + seed = Hash.randomSeed(); + } + alone = null; + t2count = null; + t2hash = null; + + for (int i = reverseOrderPos - 1; i >= 0; i--) { + long hash = reverseOrder[i]; + int found = reverseH[i]; + short xor2 = fingerprint(hash); + h012[0] = getHashFromHash(hash, 0); + h012[1] = getHashFromHash(hash, 1); + h012[2] = getHashFromHash(hash, 2); + h012[3] = h012[0]; + h012[4] = h012[1]; + fingerprints[h012[found]] = (short) (xor2 ^ fingerprints[h012[found + 1]] ^ fingerprints[h012[found + 2]]); + } + } + + @Override + public boolean mayContain(long key) { + long hash = Hash.hash64(key, seed); + short f = fingerprint(hash); + int h0 = Hash.reduce((int) (hash >>> 32), segmentCountLength); + int h1 = h0 + segmentLength; + int h2 = h1 + segmentLength; + long hh = hash; + h1 ^= (int) ((hh >> 18) & segmentLengthMask); + h2 ^= (int) ((hh) & segmentLengthMask); + f ^= fingerprints[h0] ^ fingerprints[h1] ^ fingerprints[h2]; + return (f & 0xffff) == 0; + } + + @Override + public String toString() { + return "segmentLength " + segmentLength + " segmentCount " + segmentCount; + } + + int getHashFromHash(long hash, int index) { + long h = Hash.reduce((int) (hash >>> 32), segmentCountLength); + // long h = Hash.multiplyHighUnsigned(hash, segmentCountLength); + h += index * segmentLength; + // keep the lower 36 bits + long hh = hash & ((1L << 36) - 1); + // index 0: right shift by 36; index 1: right shift by 18; index 2: no shift + h ^= (int) ((hh >>> (36 - 18 * index)) & segmentLengthMask); + return (int) h; + } + + private short fingerprint(long hash) { + return (short) hash; + } + +} \ No newline at end of file diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple.java b/fastfilter/src/main/java/org/fastfilter/xor/XorSimple.java deleted file mode 100644 index 3165e94..0000000 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.fastfilter.xor; - -import java.util.Random; - -import org.fastfilter.Filter; -import org.fastfilter.utils.Hash; - -public class XorSimple implements Filter { - - private long seed; - private byte[] data; - int blockLength; - - public long getBitCount() { - return data.length * 8; - } - - public static XorSimple construct(long[] keys) { - return new XorSimple(keys); - } - - XorSimple(long[] keys) { - blockLength = (int) ((1.23 * keys.length) + 32) / 3; - data = new byte[3 * blockLength]; - while (true) { - seed = new Random().nextLong(); - long[] stack = new long[keys.length * 2]; - if (map(keys, seed, stack)) { - assign(stack, data); - return; - } - } - } - - boolean map(long[] keys, long seed, long[] stack) { - int[] C = new int[3 * blockLength]; - long[] H = new long[3 * blockLength]; - for (long k : keys) { - long x = Hash.hash64(k, seed); - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]++; - H[index] ^= x; - } - } - int[] Q = new int[3 * blockLength]; - int qi = 0; - for (int i = 0; i < C.length; i++) { - if (C[i] == 1) { - Q[qi++] = i; - } - } - int si = 0; - while (si < 2 * keys.length) { - int i = Q[--qi]; - if (C[i] == 1) { - long x = H[i]; - stack[si++] = x; - stack[si++] = i; - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]--; - if (C[index] == 1) { - Q[qi++] = index; - } - H[index] ^= x; - } - } - } - return si == 2 * keys.length; - } - - void assign(long[] stack, byte[] b) { - for(int stackPos = stack.length; stackPos > 0;) { - int index = (int) stack[--stackPos]; - long x = stack[--stackPos]; - b[index] = (byte) (fingerprint(x) ^ b[h(x, 0)] ^ b[h(x, 1)] ^ b[h(x, 2)]); - } - } - - int h(long x, int index) { - return Hash.reduce((int) Long.rotateLeft(x, index * 21), blockLength) + index * blockLength; - } - - @Override - public boolean mayContain(long key) { - long x = Hash.hash64(key, seed); - return fingerprint(x) == (data[h(x, 0)] ^ data[h(x, 1)] ^ data[h(x, 2)]); - } - - private byte fingerprint(long x) { - return (byte) x; - } - -} diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple2.java b/fastfilter/src/main/java/org/fastfilter/xor/XorSimple2.java deleted file mode 100644 index 8dd5339..0000000 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple2.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.fastfilter.xor; - -import org.fastfilter.utils.Hash; - -/** - * The same as xor simple, but with the alternative construction that ensures - * most empty entries are in the third part of the table. - */ -public class XorSimple2 extends XorSimple { - - public static XorSimple construct(long[] keys) { - return new XorSimple(keys); - } - - public XorSimple2(long[] keys) { - super(keys); - } - - boolean map(long[] keys, long seed, long[] stack) { - int[] C = new int[3 * blockLength]; - long[] H = new long[3 * blockLength]; - for (long k : keys) { - long x = Hash.hash64(k, seed); - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]++; - H[index] ^= x; - } - } - int[][] Q = new int[3][blockLength]; - int[] qi = new int[3]; - for (int i = 0; i < C.length; i++) { - if (C[i] == 1) { - int b = i / blockLength; - Q[b][qi[b]++] = i; - } - } - int si = 0; - while (si < 2 * keys.length && qi[0] > 0 || qi[1] > 0 || qi[2] > 0) { - int i; - if (qi[0] > 0) { - i = Q[0][--qi[0]]; - } else if (qi[1] > 0) { - i = Q[1][--qi[1]]; - } else if (qi[2] > 0) { - i = Q[2][--qi[2]]; - } else { - throw new AssertionError(); - } - if (C[i] == 1) { - long x = H[i]; - stack[si++] = x; - stack[si++] = i; - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]--; - if (C[index] == 1) { - int b = index / blockLength; - Q[b][qi[b]++] = index; - } - H[index] ^= x; - } - } - } - return si == 2 * keys.length; - } - -} diff --git a/fastfilter/src/test/java/org/fastfilter/RegressionTests.java b/fastfilter/src/test/java/org/fastfilter/RegressionTests.java index 289b5fd..12a160e 100644 --- a/fastfilter/src/test/java/org/fastfilter/RegressionTests.java +++ b/fastfilter/src/test/java/org/fastfilter/RegressionTests.java @@ -22,7 +22,6 @@ public static Object[][] regressionCases() { {SUCCINCT_COUNTING_BLOCKED_BLOOM, 6049486880293779298L, new long[]{1, 2, 3}, 8}, {SUCCINCT_COUNTING_BLOCKED_BLOOM, 353772444652436712L, new long[]{5828366214313827392L, -8467365400393984494L, -424469057572555653L}, 8}, // actually this one is impossible to reproduce because of the volatile seed - {XOR_SIMPLE, 6831634639270950343L, new long[]{1, 2, 3}, 8}, {CUCKOO_8, 6335419348330489927L, new long[]{1, 2, 3}, 8}, {CUCKOO_16, -9087718164446355442L, new long[]{1, 2, 3}, 8}, {CUCKOO_PLUS_8, -4031187722136552688L, new long[]{2173645522219008926L, 589862361776609381L, -1776331367981897399L, -7505626095864333717L, 6968992741301426055L, -3110009760358584538L, diff --git a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java index eb91303..036df06 100644 --- a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java +++ b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java @@ -15,8 +15,6 @@ import org.fastfilter.xor.Xor8; import org.fastfilter.xor.XorBinaryFuse8; import org.fastfilter.xor.XorBinaryFuse16; -import org.fastfilter.xor.XorSimple; -import org.fastfilter.xor.XorSimple2; import org.fastfilter.xorplus.XorPlus8; /** @@ -65,24 +63,6 @@ public Filter construct(long[] keys, int setting) { return SuccinctCountingBlockedBloomRanked.construct(keys, setting); } }, - XOR_SIMPLE { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple.construct(keys); - } - }, - XOR_SIMPLE_2 { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple2.construct(keys); - } - }, - XOR_8 { - @Override - public Filter construct(long[] keys, int setting) { - return Xor8.construct(keys); - } - }, XOR_16 { @Override public Filter construct(long[] keys, int setting) { diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java index 4864316..10cb13e 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java @@ -14,8 +14,6 @@ public void small() { Xor16.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); XorBinaryFuse8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); XorBinaryFuse16.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); - XorSimple.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); - XorSimple2.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); } @Test diff --git a/jmh/src/main/java/org/fastfilter/ConstructionState.java b/jmh/src/main/java/org/fastfilter/ConstructionState.java index cfb9ea7..1d7cd72 100644 --- a/jmh/src/main/java/org/fastfilter/ConstructionState.java +++ b/jmh/src/main/java/org/fastfilter/ConstructionState.java @@ -19,8 +19,6 @@ public class ConstructionState { "BLOCKED_BLOOM", "SUCCINCT_COUNTING_BLOCKED_BLOOM", "SUCCINCT_COUNTING_BLOCKED_BLOOM_RANKED", - "XOR_SIMPLE", - "XOR_SIMPLE_2", "XOR_8", "XOR_16", "XOR_BINARY_FUSE_8", From 6b0c3c05353542ea66636732b9a8db5e3fbb2a5e Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Fri, 2 Jan 2026 11:25:30 -0500 Subject: [PATCH 05/38] Update Xor binary fuse filter description in README Corrected the bit variants for the Xor binary fuse filter and clarified space requirements. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e4f5da..209d396 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The following filter types are currently implemented: * Xor filter: 8 and 16 bit variants; needs less space than cuckoo filters, with faster lookup * Xor+ filter: 8 and 16 bit variants; compressed xor filter -* Xor binary fuse filter: 8 and 32 bit variants; needs less space than xor filters, with faster lookup +* Xor binary fuse filter: 8. 16 and 32 bit variants; needs less space than xor filters (for large filters), with faster lookup * Cuckoo filter: 8 and 16 bit variants; uses cuckoo hashing to store fingerprints * Cuckoo+ filter: 8 and 16 bit variants, need a bit less space than regular cuckoo filters * Bloom filter: the 'standard' algorithm From 64f7e871f03aa7a9ae6d0df1287b40ae26cd5f6b Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:08:11 +0100 Subject: [PATCH 06/38] Clarify data format --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 209d396..c604a75 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ When using Maven: * Rust: [1](https://github.com/bnclabs/xorfilter), [2](https://github.com/codri/xorfilter-rs), [3](https://github.com/Polochon-street/rustxorfilter) * [C#](https://github.com/jonmat/FastIndex) +Note that the data format in other implementations may not match the data format in Java. + ## Password Lookup Tool Included is a tool to build a filter from a list of known password (hashes), and a tool to do lookups. That way, the password list can be queried locally, without requiring a large file. The filter is only 650 MB, instead of the original file which is 11 GB. At the cost of some false positives (unknown passwords reported as known, with about 1% probability). From f4bca90459be8740d688d23b1fe1582b35fb5a33 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:14:29 +0100 Subject: [PATCH 07/38] Update pom.xml: fix scm section --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 101feba..fc3dc18 100644 --- a/pom.xml +++ b/pom.xml @@ -46,8 +46,8 @@ - scm:git:git://git@github.com:FastFilter/fastfilter_java.git - scm:git:ssh://git@github.com:FastFilter/fastfilter_java.git + scm:git:git@github.com:FastFilter/fastfilter_java.git + scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master From 2dc5a483951f66660de5a68dab3a0b832fcda4b9 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:15:17 +0100 Subject: [PATCH 08/38] [maven-release-plugin] prepare release fastfilter_java-1.0.3 --- fastfilter/pom.xml | 6 ++---- jmh/pom.xml | 6 ++---- pom.xml | 9 ++++----- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index d1d96bb..af13842 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -1,11 +1,9 @@ - + io.github.fastfilter fastfilter_java - 1.0.3-SNAPSHOT + 1.0.3 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index 7c7392a..a2ac1fe 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -1,11 +1,9 @@ - + io.github.fastfilter fastfilter_java - 1.0.3-SNAPSHOT + 1.0.3 4.0.0 diff --git a/pom.xml b/pom.xml index fc3dc18..e33b32a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,13 +1,11 @@ - + 4.0.0 io.github.fastfilter fastfilter_java pom - 1.0.3-SNAPSHOT + 1.0.3 fastfilter jmh @@ -49,7 +47,8 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - + fastfilter_java-1.0.3 + From acce992e26d0c7dc09b832ea5c61d1c838be7e39 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:15:21 +0100 Subject: [PATCH 09/38] [maven-release-plugin] prepare for next development iteration --- fastfilter/pom.xml | 2 +- jmh/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index af13842..b998fea 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.3 + 1.0.4-SNAPSHOT 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index a2ac1fe..a61d871 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.3 + 1.0.4-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index e33b32a..3369a5f 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.github.fastfilter fastfilter_java pom - 1.0.3 + 1.0.4-SNAPSHOT fastfilter jmh @@ -47,7 +47,7 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - fastfilter_java-1.0.3 + HEAD From 0e17b57b308f077282fb2be384221a40cba54523 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:32:13 +0100 Subject: [PATCH 10/38] Update pom.xml: fix scm section --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3369a5f..e120396 100644 --- a/pom.xml +++ b/pom.xml @@ -53,11 +53,11 @@ ossrh - https://s01.oss.sonatype.org/content/repositories/snapshots + https://ossrh-staging-api.central.sonatype.com/content/repositories/snapshots ossrh - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + https://ossrh-staging-api.central.sonatype.com/service/local/staging/deploy/maven2/ From f568fe7769dd3346eaca42cecdd8d78c8def6869 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:33:34 +0100 Subject: [PATCH 11/38] [maven-release-plugin] prepare release fastfilter_java-1.0.4 --- fastfilter/pom.xml | 2 +- jmh/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index b998fea..588c761 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.4-SNAPSHOT + 1.0.4 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index a61d871..576cce2 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.4-SNAPSHOT + 1.0.4 4.0.0 diff --git a/pom.xml b/pom.xml index e120396..5883284 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.github.fastfilter fastfilter_java pom - 1.0.4-SNAPSHOT + 1.0.4 fastfilter jmh @@ -47,7 +47,7 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - HEAD + fastfilter_java-1.0.4 From 8bf16c4870784b8efdc73af32f717f812066ff0e Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:33:37 +0100 Subject: [PATCH 12/38] [maven-release-plugin] prepare for next development iteration --- fastfilter/pom.xml | 2 +- jmh/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index 588c761..2e84df2 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.4 + 1.0.5-SNAPSHOT 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index 576cce2..5960043 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.4 + 1.0.5-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 5883284..fbf3342 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.github.fastfilter fastfilter_java pom - 1.0.4 + 1.0.5-SNAPSHOT fastfilter jmh @@ -47,7 +47,7 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - fastfilter_java-1.0.4 + HEAD From e1c40b11521d51921e4e08c8b43ebbff2976f00a Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Jan 2026 09:35:19 +0100 Subject: [PATCH 13/38] Update version number --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c604a75..c8d4ab3 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ When using Maven: io.github.fastfilter fastfilter - 1.0.3 + 1.0.4 # Other Xor Filter Implementations From b1b347d50aea5931adaddceaf17cad146d21ab09 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 6 Jan 2026 09:13:01 -0500 Subject: [PATCH 14/38] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c8d4ab3..2b34f21 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The following additional types are implemented, but less tested: ## Reference -* Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics (to appear). DOI: 10.1145/3510449 +* Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 * Thomas Mueller Graf, Daniel Lemire, [Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters](https://arxiv.org/abs/1912.08258), Journal of Experimental Algorithmics 25 (1), 2020. DOI: 10.1145/3376122 ## Usage @@ -46,6 +46,7 @@ When using Maven: * [Python](https://github.com/FastFilter/pyfusefilter) * Rust: [1](https://github.com/bnclabs/xorfilter), [2](https://github.com/codri/xorfilter-rs), [3](https://github.com/Polochon-street/rustxorfilter) * [C#](https://github.com/jonmat/FastIndex) +* [Java C wrapper](https://github.com/FastFilter/jfusebin) Note that the data format in other implementations may not match the data format in Java. From a808079b75245b1f35120e25a907cd816ef9679b Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 6 Jan 2026 09:51:18 -0500 Subject: [PATCH 15/38] adding an XOR/Fuse benchmark --- README.md | 67 +++++++++ jmh/pom.xml | 35 +++++ .../java/org/fastfilter/FilterBenchmark.java | 130 ++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 jmh/src/main/java/org/fastfilter/FilterBenchmark.java diff --git a/README.md b/README.md index 2b34f21..078b348 100644 --- a/README.md +++ b/README.md @@ -87,5 +87,72 @@ and with less than 1% probability "Found" or "Found; common". Internally, the tool uses a xor+ filter (see above) with 8 bits per fingerprint. Actually, 1024 smaller filters (segments) are made, the segment id being the highest 10 bits of the key. The lowest bit of the key is set to either 0 (regular) or 1 (common), and so two lookups are made per password. Because of that, the false positive rate is twice of what it would be with just one lookup (0.0078 instead of 0.0039). A regular Bloom filter with the same guarantees would be ~760 MB. For each lookup, one filter segment (so, less than 1 MB) are read from the file. +## Benchmarks + +The project includes JMH (Java Microbenchmark Harness) benchmarks to measure the performance of the filters. + +### Running Benchmarks + +#### Option 1: Run via Maven (recommended) + +To run the benchmarks directly from Maven (with minimal iterations for quick testing): + + mvn -pl jmh clean package exec:exec@run-benchmarks + +For full benchmarks, modify the pom.xml or run the JAR manually with custom parameters. + +This will compile and execute the JMH benchmarks for the XOR filters (XOR_8, XOR_16, XOR_BINARY_FUSE_8, XOR_BINARY_FUSE_16). + +#### Option 2: Run the JAR manually + +First, build the project: + + mvn clean package + +Then run the benchmarks: + + java -jar jmh/target/benchmarks.jar org.fastfilter.FilterBenchmark + +To run benchmarks for a specific filter type: + + java -jar jmh/target/benchmarks.jar org.fastfilter.FilterBenchmark -p filterType=XOR_BINARY_FUSE_8 + +Available filter types: `XOR_8`, `XOR_16`, `XOR_BINARY_FUSE_8`, `XOR_BINARY_FUSE_16`. + +### Benchmark Details + +The benchmarks measure: +- Average time per operation (nanoseconds) for lookups of existing and non-existing keys +- Throughput (operations per second) for the same operations +- False positive rate validation + + +Possible results: + +``` + +Benchmark (filterType) Mode Cnt Score Error Units +FilterBenchmark.benchmarkContainsExistingThroughput XOR_8 thrpt 412364492,755 ops/s +FilterBenchmark.benchmarkContainsExistingThroughput XOR_16 thrpt 397627818,837 ops/s +FilterBenchmark.benchmarkContainsExistingThroughput XOR_BINARY_FUSE_8 thrpt 516262004,459 ops/s +FilterBenchmark.benchmarkContainsExistingThroughput XOR_BINARY_FUSE_16 thrpt 489256453,340 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_8 thrpt 429856367,135 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_16 thrpt 441042890,257 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_BINARY_FUSE_8 thrpt 533609392,046 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_BINARY_FUSE_16 thrpt 540058414,150 ops/s +FilterBenchmark.benchmarkContainsExisting XOR_8 avgt 2,475 ns/op +FilterBenchmark.benchmarkContainsExisting XOR_16 avgt 2,522 ns/op +FilterBenchmark.benchmarkContainsExisting XOR_BINARY_FUSE_8 avgt 1,965 ns/op +FilterBenchmark.benchmarkContainsExisting XOR_BINARY_FUSE_16 avgt 2,060 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_8 avgt 2,347 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_16 avgt 2,295 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_BINARY_FUSE_8 avgt 1,892 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_BINARY_FUSE_16 avgt 1,903 ns/op +``` + +This indicates that we can issue about half a billion queries per second, and sustain a rate of about 2 ns per query. + +The benchmarks use 1,000,000 keys by default. You can modify the `NUM_KEYS` constant in `FilterBenchmark.java` for smaller/larger test sets. + diff --git a/jmh/pom.xml b/jmh/pom.xml index 5960043..4be23ea 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -39,6 +39,13 @@ true ${maven.compiler.source} ${maven.compiler.target} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + true true true @@ -67,6 +74,34 @@ + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + run-benchmarks + + exec + + + java + + -jar + ${project.build.directory}/benchmarks.jar + org.fastfilter.FilterBenchmark + -f + 1 + -wi + 1 + -i + 1 + + + + + diff --git a/jmh/src/main/java/org/fastfilter/FilterBenchmark.java b/jmh/src/main/java/org/fastfilter/FilterBenchmark.java new file mode 100644 index 0000000..9f0be05 --- /dev/null +++ b/jmh/src/main/java/org/fastfilter/FilterBenchmark.java @@ -0,0 +1,130 @@ +package org.fastfilter; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import org.fastfilter.Filter; +import org.fastfilter.xor.Xor8; +import org.fastfilter.xor.Xor16; +import org.fastfilter.xor.XorBinaryFuse8; +import org.fastfilter.xor.XorBinaryFuse16; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +@State(Scope.Benchmark) +public class FilterBenchmark { + + @Param({"XOR_8", "XOR_16", "XOR_BINARY_FUSE_8", "XOR_BINARY_FUSE_16"}) + public String filterType; + + private Filter filter; + private long[] testKeys; + private final int NUM_KEYS = 1_000_000; + + @Setup + public void setup() { + // Create 1,000,000 keys (even numbers) + testKeys = new long[NUM_KEYS]; + for (int i = 0; i < testKeys.length; i++) { + testKeys[i] = (long) i * 2L; // even numbers + } + + try { + switch (filterType) { + case "XOR_8": + filter = Xor8.construct(testKeys); + break; + case "XOR_16": + filter = Xor16.construct(testKeys); + break; + case "XOR_BINARY_FUSE_8": + filter = XorBinaryFuse8.construct(testKeys); + break; + case "XOR_BINARY_FUSE_16": + filter = XorBinaryFuse16.construct(testKeys); + break; + default: + throw new IllegalArgumentException("Unknown filter type: " + filterType); + } + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @TearDown + public void tearDown() { + filter = null; + testKeys = null; + } + + @Benchmark + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsExisting(Blackhole blackhole) throws Throwable { + for (long key : testKeys) { + if (!filter.mayContain(key)) { + throw new RuntimeException("Key should exist: " + key); + } + } + } + + @Benchmark + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsNonExisting(Blackhole blackhole) throws Throwable { + int fp = 0; + for (int i = 0; i < testKeys.length; i++) { + long key = (long) i * 2L + 1L; // odd numbers + if (filter.mayContain(key)) { + fp++; + } + } + if (fp > 10000) { + throw new RuntimeException("Too many false positives: " + fp); + } + } + + @Benchmark + @BenchmarkMode(Mode.Throughput) + @OutputTimeUnit(TimeUnit.SECONDS) + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsExistingThroughput(Blackhole blackhole) throws Throwable { + for (long key : testKeys) { + if (!filter.mayContain(key)) { + throw new RuntimeException("Key should exist: " + key); + } + } + } + + @Benchmark + @BenchmarkMode(Mode.Throughput) + @OutputTimeUnit(TimeUnit.SECONDS) + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsNonExistingThroughput(Blackhole blackhole) throws Throwable { + int fp = 0; + for (int i = 0; i < testKeys.length; i++) { + long key = (long) i * 2L + 1L; // odd numbers + if (filter.mayContain(key)) { + fp++; + } + } + if (fp > 10000) { + throw new RuntimeException("Too many false positives: " + fp); + } + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(FilterBenchmark.class.getSimpleName()) + .build(); + + new Runner(opt).run(); + } +} From 9dc3f7b28cad1d82eaaff2bf12a20e27c601d400 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 6 Jan 2026 10:29:23 -0500 Subject: [PATCH 16/38] Update Java setup to JDK 22 in workflow --- .github/workflows/java.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 8d44d9d..5176ded 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -8,10 +8,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 17 - uses: actions/setup-java@v1 + - name: Set up JDK 22 + uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 22 + distribution: 'temurin' - name: Build run: mvn package - name: Test From 0ccdf97c5b1c78371216ef0ad5bb3b6b38ca355d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 6 Jan 2026 10:29:33 -0500 Subject: [PATCH 17/38] Update CI workflow to use Java 22 --- .github/workflows/java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 5176ded..0f1a00d 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -1,4 +1,4 @@ -name: Java 17 CI +name: Java 22 CI on: [push,pull_request] From 520e7d385a1f2d52cc48b571805f57530bc1a794 Mon Sep 17 00:00:00 2001 From: tinder-igorsokolov Date: Wed, 7 Jan 2026 10:25:15 -0800 Subject: [PATCH 18/38] Added serialization/deserialization capabilities to the XorBinaryFuse16 --- .../src/main/java/org/fastfilter/Filter.java | 22 +- .../org/fastfilter/xor/XorBinaryFuse16.java | 75 ++++- .../org/fastfilter/xor/SerializationTest.java | 266 ++++++++++++++++++ 3 files changed, 350 insertions(+), 13 deletions(-) create mode 100644 fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java diff --git a/fastfilter/src/main/java/org/fastfilter/Filter.java b/fastfilter/src/main/java/org/fastfilter/Filter.java index 5eedbcb..05bfe43 100644 --- a/fastfilter/src/main/java/org/fastfilter/Filter.java +++ b/fastfilter/src/main/java/org/fastfilter/Filter.java @@ -1,5 +1,7 @@ package org.fastfilter; +import java.nio.ByteBuffer; + /** * An approximate membership filter. */ @@ -14,7 +16,7 @@ public interface Filter { boolean mayContain(long key); /** - * Get the number of bits in thhe filter. + * Get the number of bits in the filter. * * @return the number of bits */ @@ -65,4 +67,22 @@ default long cardinality() { return -1; } + /** + * Get the serialized size of the filter. + * + * @return the size in bytes + */ + default int getSerializedSize() { + return -1; + } + + /** + * Serializes the filter state into the provided {@code ByteBuffer}. + * + * @param buffer the byte buffer where the serialized state of the filter will be written + * @throws UnsupportedOperationException if the operation is not supported by the filter implementation + */ + default void serialize(ByteBuffer buffer) { + throw new UnsupportedOperationException(); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index aad3c77..4d1c0fb 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -1,5 +1,7 @@ package org.fastfilter.xor; +import java.lang.reflect.Constructor; +import java.nio.ByteBuffer; import java.util.Arrays; import org.fastfilter.Filter; @@ -20,19 +22,25 @@ public class XorBinaryFuse16 implements Filter { private final short[] fingerprints; private long seed; - public XorBinaryFuse16(int segmentCount, int segmentLength) { + private XorBinaryFuse16(int segmentCount, int segmentLength, long seed, short[] fingerprints) { if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) { throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength); } if (segmentCount <= 0) { throw new IllegalArgumentException("Illegal segment count: " + segmentCount); } - this.segmentLength = segmentLength; + this.segmentCount = segmentCount; - this.segmentLengthMask = segmentLength - 1; this.segmentCountLength = segmentCount * segmentLength; - this.arrayLength = (segmentCount + ARITY - 1) * segmentLength; - this.fingerprints = new short[arrayLength]; + this.segmentLength = segmentLength; + this.segmentLengthMask = segmentLength - 1; + this.arrayLength = fingerprints.length; + this.fingerprints = fingerprints; + this.seed = seed; + } + + public XorBinaryFuse16(int segmentCount, int segmentLength) { + this(segmentCount, segmentLength, 0L, new short[(segmentCount + ARITY - 1) * segmentLength]); } public long getBitCount() { @@ -143,8 +151,7 @@ private void addAll(long[] keys) { countMask |= t2count[index]; } } - startPos = null; - if (countMask < 0) { + if (countMask < 0) { // we have a possible counter overflow continue mainloop; } @@ -210,11 +217,8 @@ private void addAll(long[] keys) { // use a new random numbers seed = Hash.randomSeed(); } - alone = null; - t2count = null; - t2hash = null; - for (int i = reverseOrderPos - 1; i >= 0; i--) { + for (int i = reverseOrderPos - 1; i >= 0; i--) { long hash = reverseOrder[i]; int found = reverseH[i]; short xor2 = fingerprint(hash); @@ -261,4 +265,51 @@ private short fingerprint(long hash) { return (short) hash; } -} \ No newline at end of file + @Override + public int getSerializedSize() { + return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Short.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(segmentLength); + buffer.putInt(segmentCountLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + for (final short fp : fingerprints) { + buffer.putShort(fp); + } + } + + public static XorBinaryFuse16 deserialize(ByteBuffer buffer) { + // Check minimum size for header (2 ints + 1 long + 1 int for length) + if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int segmentLength = buffer.getInt(); + final int segmentCountLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Short.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = buffer.getShort(); + } + + // Calculate segmentCount from segmentCountLength and segmentLength + final int segmentCount = segmentCountLength / segmentLength; + + return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints); + } +} diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java new file mode 100644 index 0000000..d2113e1 --- /dev/null +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -0,0 +1,266 @@ +package org.fastfilter.xor; + +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SerializationTest { + + @Test + public void shouldSerializeAndDeserializeEmptyFilter() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L}; + final var originalFilter = XorBinaryFuse16.construct(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized filter", + deserializedFilter.mayContain(key)); + } + } + + @Test + public void shouldSerializeAndDeserializeSmallFilter() { + // Arrange + final var keys = new long[]{100L, 200L, 300L, 400L, 500L, 600L, 700L, 800L, 900L, 1000L}; + final var originalFilter = XorBinaryFuse16.construct(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 50L should not be in filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 1500L should not be in filter", deserializedFilter.mayContain(1500L)); + } + + @Test + public void shouldSerializeAndDeserializeLargeFilter() { + // Arrange + final int size = 10000; + final var keys = new long[size]; + for (int i = 0; i < size; i++) { + keys[i] = i * 100L; + } + final var originalFilter = XorBinaryFuse16.construct(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + + // Assert + for (int i = 0; i < size; i++) { + final long key = i * 100L; + assertTrue("Key " + key + " should be present in deserialized filter", + deserializedFilter.mayContain(key)); + } + // Test some keys that should not be in the filter + assertFalse("Key 1L should not be in filter", deserializedFilter.mayContain(1L)); + assertFalse("Key 50L should not be in filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 99L should not be in filter", deserializedFilter.mayContain(99L)); + } + + @Test + public void shouldPreserveFilterCharacteristicsAfterDeserialization() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L}; + final var originalFilter = XorBinaryFuse16.construct(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + + // Assert + assertEquals("Bit count should be preserved", + originalFilter.getBitCount(), deserializedFilter.getBitCount()); + assertEquals("Serialized size should be preserved", + originalFilter.getSerializedSize(), deserializedFilter.getSerializedSize()); + assertEquals("String representation should match", + originalFilter.toString(), deserializedFilter.toString()); + } + + @Test + public void shouldHandleMultipleSerializationRounds() { + // Arrange + final var keys = new long[]{10L, 20L, 30L, 40L, 50L}; + final var originalFilter = XorBinaryFuse16.construct(keys); + final var buffer1 = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act - First round + originalFilter.serialize(buffer1); + buffer1.flip(); + final var filter1 = XorBinaryFuse16.deserialize(buffer1); + + // Act - Second round + final var buffer2 = ByteBuffer.allocate(filter1.getSerializedSize()); + filter1.serialize(buffer2); + buffer2.flip(); + final var filter2 = XorBinaryFuse16.deserialize(buffer2); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present after first deserialization", + filter1.mayContain(key)); + assertTrue("Key " + key + " should be present after second deserialization", + filter2.mayContain(key)); + } + } + + @Test + public void shouldThrowExceptionWhenSerializeBufferTooSmall() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L}; + final var filter = XorBinaryFuse16.construct(keys); + final var smallBuffer = ByteBuffer.allocate(filter.getSerializedSize() - 1); + + // Act & Assert + try { + filter.serialize(smallBuffer); + fail("Should have thrown IllegalArgumentException for buffer too small"); + } catch (IllegalArgumentException e) { + assertEquals("Buffer too small", e.getMessage()); + } + } + + @Test + public void shouldThrowExceptionWhenDeserializeBufferTooSmall() { + // Arrange + final var tooSmallBuffer = ByteBuffer.allocate(10); + + // Act & Assert + try { + XorBinaryFuse16.deserialize(tooSmallBuffer); + fail("Should have thrown IllegalArgumentException for buffer too small"); + } catch (IllegalArgumentException e) { + assertEquals("Buffer too small", e.getMessage()); + } + } + + @Test + public void shouldHandleFilterWithSequentialKeys() { + // Arrange + final int size = 1000; + final var keys = new long[size]; + for (int i = 0; i < size; i++) { + keys[i] = i; + } + final var originalFilter = XorBinaryFuse16.construct(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + + // Assert + for (int i = 0; i < size; i++) { + assertTrue("Sequential key " + i + " should be present", + deserializedFilter.mayContain(i)); + } + assertFalse("Key outside range should not be in filter", + deserializedFilter.mayContain(size + 1000)); + } + + @Test + public void shouldHandleFilterWithRandomLargeKeys() { + // Arrange + final var keys = new long[]{ + Long.MAX_VALUE - 1, + Long.MAX_VALUE - 100, + Long.MAX_VALUE - 1000, + Long.MAX_VALUE / 2, + Long.MAX_VALUE / 3 + }; + final var originalFilter = XorBinaryFuse16.construct(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Large key " + key + " should be present", + deserializedFilter.mayContain(key)); + } + } + + @Test + public void shouldCorrectlyCalculateSerializedSize() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L}; + final var filter = XorBinaryFuse16.construct(keys); + final int expectedSizeInBytes = filter.getSerializedSize(); + final var buffer = ByteBuffer.allocate(expectedSizeInBytes); + + // Act + filter.serialize(buffer); + + // Assert + assertEquals("Buffer position should equal serialized size", + expectedSizeInBytes, buffer.position()); + assertEquals("Buffer should have no remaining space", + 0, buffer.remaining()); + } + + @Test + public void shouldHandleExactBufferSize() { + // Arrange + final var keys = new long[]{100L, 200L, 300L}; + final var filter = XorBinaryFuse16.construct(keys); + final var exactBuffer = ByteBuffer.allocate(filter.getSerializedSize()); + + // Act + filter.serialize(exactBuffer); + exactBuffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(exactBuffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present with exact buffer", + deserializedFilter.mayContain(key)); + } + assertEquals("No bytes should remain in buffer", 0, exactBuffer.remaining()); + } + + @Test + public void shouldHandleLargerBufferThanNeeded() { + // Arrange + final var keys = new long[]{1L, 2L, 3L}; + final var filter = XorBinaryFuse16.construct(keys); + final var largeBuffer = ByteBuffer.allocate(filter.getSerializedSize() + 1000); + + // Act + filter.serialize(largeBuffer); + largeBuffer.flip(); + final var deserializedFilter = XorBinaryFuse16.deserialize(largeBuffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present with larger buffer", + deserializedFilter.mayContain(key)); + } + } +} From ec1e7a5b17da7d3313d04f63a82134e1a0f896a5 Mon Sep 17 00:00:00 2001 From: tinder-igorsokolov Date: Wed, 7 Jan 2026 13:55:12 -0800 Subject: [PATCH 19/38] small style fixes --- .../src/main/java/org/fastfilter/utils/Hash.java | 12 ++++++++---- .../java/org/fastfilter/xor/XorBinaryFuse16.java | 10 ++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java index 6e6b02f..709c407 100644 --- a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java +++ b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java @@ -2,11 +2,15 @@ import java.util.Random; -public class Hash { +public final class Hash { + private Hash() { - private static Random random = new Random(); + } + + private static final Random random = new Random(); public static void setSeed(long seed) { + // shouldn't we use ThreadLocalRandom.current() instead? random.setSeed(seed); } @@ -23,7 +27,7 @@ public static long randomSeed() { } /** - * Shrink the hash to a value 0..n. Kind of like modulo, but using + * Shrink the hash to value 0..n. Kind of like modulo, but using * multiplication and shift, which are faster to compute. * * @param hash the hash @@ -37,7 +41,7 @@ public static int reduce(int hash, int n) { /** * Multiply two unsigned 64-bit values. - * See https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8188044 + * See JDK-8188044 * * @param a the first value * @param b the second value diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index 4d1c0fb..341034d 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -48,7 +48,7 @@ public long getBitCount() { } static int calculateSegmentLength(int arity, int size) { - int segmentLength; + final int segmentLength; if (arity == 3) { segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(3.33) + 2.11); } else if (arity == 4) { @@ -61,7 +61,7 @@ static int calculateSegmentLength(int arity, int size) { } static double calculateSizeFactor(int arity, int size) { - double sizeFactor; + final double sizeFactor; if (arity == 3) { sizeFactor = Math.max(1.125, 0.875 + 0.25 * Math.log(1000000) / Math.log(size)); } else if (arity == 4) { @@ -209,12 +209,10 @@ private void addAll(long[] keys) { // if construction doesn't succeed eventually, // then there is likely a problem with the hash function // let us not crash the system: - for(int i = 0; i < fingerprints.length; i++) { - fingerprints[i] = (short)0xFFFF; - } + Arrays.fill(fingerprints, (short) 0xFFFF); return; } - // use a new random numbers + // use a new random number seed = Hash.randomSeed(); } From ea80a7da42020301342a44f7be26200857fbeda2 Mon Sep 17 00:00:00 2001 From: tinder-igorsokolov Date: Thu, 8 Jan 2026 12:02:34 -0500 Subject: [PATCH 20/38] reverting unrelated changes --- .../src/main/java/org/fastfilter/utils/Hash.java | 11 +++-------- .../java/org/fastfilter/xor/XorBinaryFuse16.java | 12 +++++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java index 709c407..1eeb8de 100644 --- a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java +++ b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java @@ -2,15 +2,10 @@ import java.util.Random; -public final class Hash { - private Hash() { - - } - +public class Hash { private static final Random random = new Random(); public static void setSeed(long seed) { - // shouldn't we use ThreadLocalRandom.current() instead? random.setSeed(seed); } @@ -27,7 +22,7 @@ public static long randomSeed() { } /** - * Shrink the hash to value 0..n. Kind of like modulo, but using + * Shrink the hash to a value 0..n. Kind of like modulo, but using * multiplication and shift, which are faster to compute. * * @param hash the hash @@ -41,7 +36,7 @@ public static int reduce(int hash, int n) { /** * Multiply two unsigned 64-bit values. - * See JDK-8188044 + * See https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8188044 * * @param a the first value * @param b the second value diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index 341034d..ca8a235 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -1,9 +1,7 @@ package org.fastfilter.xor; -import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.Arrays; - import org.fastfilter.Filter; import org.fastfilter.utils.Hash; @@ -61,7 +59,7 @@ static int calculateSegmentLength(int arity, int size) { } static double calculateSizeFactor(int arity, int size) { - final double sizeFactor; + double sizeFactor; if (arity == 3) { sizeFactor = Math.max(1.125, 0.875 + 0.25 * Math.log(1000000) / Math.log(size)); } else if (arity == 4) { @@ -151,7 +149,8 @@ private void addAll(long[] keys) { countMask |= t2count[index]; } } - if (countMask < 0) { + startPos = null; + if (countMask < 0) { // we have a possible counter overflow continue mainloop; } @@ -216,7 +215,10 @@ private void addAll(long[] keys) { seed = Hash.randomSeed(); } - for (int i = reverseOrderPos - 1; i >= 0; i--) { + alone = null; + t2count = null; + t2hash = null; + for (int i = reverseOrderPos - 1; i >= 0; i--) { long hash = reverseOrder[i]; int found = reverseH[i]; short xor2 = fingerprint(hash); From 69abf1a9f7db279c7ff1665d4e5b2467c3bf9823 Mon Sep 17 00:00:00 2001 From: tinder-igorsokolov Date: Thu, 8 Jan 2026 12:04:08 -0500 Subject: [PATCH 21/38] reverting unrelated changes --- fastfilter/src/main/java/org/fastfilter/utils/Hash.java | 2 +- .../src/main/java/org/fastfilter/xor/XorBinaryFuse16.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java index 1eeb8de..57fc3fe 100644 --- a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java +++ b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java @@ -3,7 +3,7 @@ import java.util.Random; public class Hash { - private static final Random random = new Random(); + private static Random random = new Random(); public static void setSeed(long seed) { random.setSeed(seed); diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index ca8a235..db49863 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -46,7 +46,7 @@ public long getBitCount() { } static int calculateSegmentLength(int arity, int size) { - final int segmentLength; + int segmentLength; if (arity == 3) { segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(3.33) + 2.11); } else if (arity == 4) { @@ -218,6 +218,7 @@ private void addAll(long[] keys) { alone = null; t2count = null; t2hash = null; + for (int i = reverseOrderPos - 1; i >= 0; i--) { long hash = reverseOrder[i]; int found = reverseH[i]; From f786517883bc82b9dea2f69b03c0eda395601631 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Sat, 10 Jan 2026 11:16:28 +0100 Subject: [PATCH 22/38] Update README.md --- README.md | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 078b348..50ca0fd 100644 --- a/README.md +++ b/README.md @@ -27,14 +27,47 @@ The following additional types are implemented, but less tested: ## Usage -When using Maven: +### Maven + +When using Maven: The latest version, 1.0.4, is not yet available on Maven central, see [issue #48](https://github.com/FastFilter/fastfilter_java/issues/48). However, it is available at https://jitpack.io/: + + + + jitpack.io + https://jitpack.io + + - io.github.fastfilter - fastfilter - 1.0.4 + io.github.fastfilter + fastfilter + 1.0.4 +### Gradle + + repositories { + mavenCentral() + maven { + url 'https://jitpack.io' + } + } + + dependencies { + implementation 'io.github.fastfilter.fastfilter_java:fastfilter:1.0.4' + } + +### Maven Central (version 1.0.2) + +The older version, 1.0.2, is available on Maven central. + + + io.github.fastfilter + fastfilter + 1.0.2 + + + # Other Xor Filter Implementations * [C](https://github.com/FastFilter/xor_singleheader) From 1cfc00bc2e515dddfd6cfb2149f0065299e8ad07 Mon Sep 17 00:00:00 2001 From: tinder-igorsokolov Date: Mon, 12 Jan 2026 10:57:27 -0500 Subject: [PATCH 23/38] added serialization / deserialization to Xor8, Xor16, XorBinaryFuse8, XorBinaryFuse32 --- .../main/java/org/fastfilter/xor/Xor16.java | 53 ++++++++ .../main/java/org/fastfilter/xor/Xor8.java | 48 +++++++ .../org/fastfilter/xor/XorBinaryFuse32.java | 64 ++++++++- .../org/fastfilter/xor/XorBinaryFuse8.java | 60 ++++++++- .../org/fastfilter/xor/SerializationTest.java | 126 +++++++++++------- 5 files changed, 294 insertions(+), 57 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java index bca40a6..8cdc4d8 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java @@ -1,5 +1,7 @@ package org.fastfilter.xor; +import java.nio.ByteBuffer; + import org.fastfilter.Filter; import org.fastfilter.utils.Hash; @@ -143,4 +145,55 @@ private int fingerprint(long hash) { return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1)); } + private Xor16(int blockLength, int bitCount, long seed, short[] fingerprints) { + this.blockLength = blockLength; + this.bitCount = bitCount; + this.seed = seed; + this.fingerprints = fingerprints; + } + + @Override + public int getSerializedSize() { + return Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Short.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(blockLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + for (final short fp : fingerprints) { + buffer.putShort(fp); + } + } + + public static Xor16 deserialize(ByteBuffer buffer) { + // Check minimum size for header (1 int + 1 long + 1 int for length) + if (buffer.remaining() < Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int blockLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Short.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = buffer.getShort(); + } + + final int bitCount = len * BITS_PER_FINGERPRINT; + + return new Xor16(blockLength, bitCount, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java index 86ac870..bb3b5ff 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java @@ -1,6 +1,7 @@ package org.fastfilter.xor; import java.io.*; +import java.nio.ByteBuffer; import org.fastfilter.Filter; import org.fastfilter.utils.Hash; @@ -187,4 +188,51 @@ public Xor8(InputStream in) { } } + private Xor8(int size, long seed, byte[] fingerprints) { + this.size = size; + this.arrayLength = getArrayLength(size); + this.bitCount = arrayLength * BITS_PER_FINGERPRINT; + this.blockLength = arrayLength / HASHES; + this.seed = seed; + this.fingerprints = fingerprints; + } + + @Override + public int getSerializedSize() { + return Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Byte.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(size); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + buffer.put(fingerprints); + } + + public static Xor8 deserialize(ByteBuffer buffer) { + // Check minimum size for header (1 int + 1 long + 1 int for length) + if (buffer.remaining() < Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int size = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Byte.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final byte[] fingerprints = new byte[len]; + buffer.get(fingerprints); + + return new Xor8(size, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java index d3f6125..760fc3f 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java @@ -1,5 +1,6 @@ package org.fastfilter.xor; +import java.nio.ByteBuffer; import java.util.Arrays; import org.fastfilter.Filter; @@ -20,19 +21,25 @@ public class XorBinaryFuse32 implements Filter { private final int[] fingerprints; private long seed; - public XorBinaryFuse32(int segmentCount, int segmentLength) { + private XorBinaryFuse32(int segmentCount, int segmentLength, long seed, int[] fingerprints) { if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) { throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength); } if (segmentCount <= 0) { throw new IllegalArgumentException("Illegal segment count: " + segmentCount); } - this.segmentLength = segmentLength; + this.segmentCount = segmentCount; - this.segmentLengthMask = segmentLength - 1; this.segmentCountLength = segmentCount * segmentLength; - this.arrayLength = (segmentCount + ARITY - 1) * segmentLength; - this.fingerprints = new int[arrayLength]; + this.segmentLength = segmentLength; + this.segmentLengthMask = segmentLength - 1; + this.arrayLength = fingerprints.length; + this.fingerprints = fingerprints; + this.seed = seed; + } + + public XorBinaryFuse32(int segmentCount, int segmentLength) { + this(segmentCount, segmentLength, 0L, new int[(segmentCount + ARITY - 1) * segmentLength]); } public long getBitCount() { @@ -261,4 +268,51 @@ private int fingerprint(long hash) { return (int) (hash ^ (hash >>> 32)); } + @Override + public int getSerializedSize() { + return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Integer.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(segmentLength); + buffer.putInt(segmentCountLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + for (final int fp : fingerprints) { + buffer.putInt(fp); + } + } + + public static XorBinaryFuse32 deserialize(ByteBuffer buffer) { + // Check minimum size for header (2 ints + 1 long + 1 int for length) + if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int segmentLength = buffer.getInt(); + final int segmentCountLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int[] fingerprints = new int[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = buffer.getInt(); + } + + // Calculate segmentCount from segmentCountLength and segmentLength + final int segmentCount = segmentCountLength / segmentLength; + + return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java index dfd5f45..ea16611 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java @@ -1,5 +1,6 @@ package org.fastfilter.xor; +import java.nio.ByteBuffer; import java.util.Arrays; import org.fastfilter.Filter; @@ -20,19 +21,25 @@ public class XorBinaryFuse8 implements Filter { private final byte[] fingerprints; private long seed; - public XorBinaryFuse8(int segmentCount, int segmentLength) { + private XorBinaryFuse8(int segmentCount, int segmentLength, long seed, byte[] fingerprints) { if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) { throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength); } if (segmentCount <= 0) { throw new IllegalArgumentException("Illegal segment count: " + segmentCount); } - this.segmentLength = segmentLength; + this.segmentCount = segmentCount; - this.segmentLengthMask = segmentLength - 1; this.segmentCountLength = segmentCount * segmentLength; - this.arrayLength = (segmentCount + ARITY - 1) * segmentLength; - this.fingerprints = new byte[arrayLength]; + this.segmentLength = segmentLength; + this.segmentLengthMask = segmentLength - 1; + this.arrayLength = fingerprints.length; + this.fingerprints = fingerprints; + this.seed = seed; + } + + public XorBinaryFuse8(int segmentCount, int segmentLength) { + this(segmentCount, segmentLength, 0L, new byte[(segmentCount + ARITY - 1) * segmentLength]); } public long getBitCount() { @@ -261,4 +268,47 @@ private byte fingerprint(long hash) { return (byte) hash; } + @Override + public int getSerializedSize() { + return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Byte.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(segmentLength); + buffer.putInt(segmentCountLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + buffer.put(fingerprints); + } + + public static XorBinaryFuse8 deserialize(ByteBuffer buffer) { + // Check minimum size for header (2 ints + 1 long + 1 int for length) + if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int segmentLength = buffer.getInt(); + final int segmentCountLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Byte.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final byte[] fingerprints = new byte[len]; + buffer.get(fingerprints); + + // Calculate segmentCount from segmentCountLength and segmentLength + final int segmentCount = segmentCountLength / segmentLength; + + return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java index d2113e1..df6a204 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -1,54 +1,88 @@ package org.fastfilter.xor; -import org.junit.Test; - -import java.nio.ByteBuffer; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.function.Function; +import org.fastfilter.Filter; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) public class SerializationTest { + private final String filterName; + private final Function constructor; + private final Function deserializer; + + public SerializationTest(String filterName, + Function constructor, + Function deserializer) { + this.filterName = filterName; + this.constructor = constructor; + this.deserializer = deserializer; + } + + @Parameters(name = "{0}") + public static List filters() { + return List.of( + new Object[] {"Xor8", (Function) Xor8::construct, + (Function) Xor8::deserialize}, + new Object[] {"Xor16", (Function) Xor16::construct, + (Function) Xor16::deserialize}, + new Object[] {"XorBinaryFuse8", (Function) XorBinaryFuse8::construct, + (Function) XorBinaryFuse8::deserialize}, + new Object[] {"XorBinaryFuse16", (Function) XorBinaryFuse16::construct, + (Function) XorBinaryFuse16::deserialize}, + new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, + (Function) XorBinaryFuse32::deserialize} + ); + } + @Test - public void shouldSerializeAndDeserializeEmptyFilter() { + public void shouldSerializeAndDeserializeSmallFilter() { // Arrange final var keys = new long[]{1L, 2L, 3L, 4L, 5L}; - final var originalFilter = XorBinaryFuse16.construct(keys); + final var originalFilter = constructor.apply(keys); final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); // Act originalFilter.serialize(buffer); buffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + final var deserializedFilter = deserializer.apply(buffer); // Assert for (final long key : keys) { - assertTrue("Key " + key + " should be present in deserialized filter", + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", deserializedFilter.mayContain(key)); } } @Test - public void shouldSerializeAndDeserializeSmallFilter() { + public void shouldSerializeAndDeserializeMediumFilter() { // Arrange final var keys = new long[]{100L, 200L, 300L, 400L, 500L, 600L, 700L, 800L, 900L, 1000L}; - final var originalFilter = XorBinaryFuse16.construct(keys); + final var originalFilter = constructor.apply(keys); final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); // Act originalFilter.serialize(buffer); buffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + final var deserializedFilter = deserializer.apply(buffer); // Assert for (final long key : keys) { - assertTrue("Key " + key + " should be present in deserialized filter", + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", deserializedFilter.mayContain(key)); } - assertFalse("Key 50L should not be in filter", deserializedFilter.mayContain(50L)); - assertFalse("Key 1500L should not be in filter", deserializedFilter.mayContain(1500L)); + assertFalse("Key 50L should not be in " + filterName + " filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 1500L should not be in " + filterName + " filter", deserializedFilter.mayContain(1500L)); } @Test @@ -59,18 +93,18 @@ public void shouldSerializeAndDeserializeLargeFilter() { for (int i = 0; i < size; i++) { keys[i] = i * 100L; } - final var originalFilter = XorBinaryFuse16.construct(keys); + final var originalFilter = constructor.apply(keys); final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); // Act originalFilter.serialize(buffer); buffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + final var deserializedFilter = deserializer.apply(buffer); // Assert for (int i = 0; i < size; i++) { final long key = i * 100L; - assertTrue("Key " + key + " should be present in deserialized filter", + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", deserializedFilter.mayContain(key)); } // Test some keys that should not be in the filter @@ -83,46 +117,44 @@ public void shouldSerializeAndDeserializeLargeFilter() { public void shouldPreserveFilterCharacteristicsAfterDeserialization() { // Arrange final var keys = new long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L}; - final var originalFilter = XorBinaryFuse16.construct(keys); + final var originalFilter = constructor.apply(keys); final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); // Act originalFilter.serialize(buffer); buffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + final var deserializedFilter = deserializer.apply(buffer); // Assert - assertEquals("Bit count should be preserved", + assertEquals("Bit count should be preserved for " + filterName, originalFilter.getBitCount(), deserializedFilter.getBitCount()); - assertEquals("Serialized size should be preserved", + assertEquals("Serialized size should be preserved for " + filterName, originalFilter.getSerializedSize(), deserializedFilter.getSerializedSize()); - assertEquals("String representation should match", - originalFilter.toString(), deserializedFilter.toString()); } @Test public void shouldHandleMultipleSerializationRounds() { // Arrange final var keys = new long[]{10L, 20L, 30L, 40L, 50L}; - final var originalFilter = XorBinaryFuse16.construct(keys); + final var originalFilter = constructor.apply(keys); final var buffer1 = ByteBuffer.allocate(originalFilter.getSerializedSize()); // Act - First round originalFilter.serialize(buffer1); buffer1.flip(); - final var filter1 = XorBinaryFuse16.deserialize(buffer1); + final var filter1 = deserializer.apply(buffer1); // Act - Second round final var buffer2 = ByteBuffer.allocate(filter1.getSerializedSize()); filter1.serialize(buffer2); buffer2.flip(); - final var filter2 = XorBinaryFuse16.deserialize(buffer2); + final var filter2 = deserializer.apply(buffer2); // Assert for (final long key : keys) { - assertTrue("Key " + key + " should be present after first deserialization", + assertTrue("Key " + key + " should be present after first deserialization of " + filterName, filter1.mayContain(key)); - assertTrue("Key " + key + " should be present after second deserialization", + assertTrue("Key " + key + " should be present after second deserialization of " + filterName, filter2.mayContain(key)); } } @@ -131,7 +163,7 @@ public void shouldHandleMultipleSerializationRounds() { public void shouldThrowExceptionWhenSerializeBufferTooSmall() { // Arrange final var keys = new long[]{1L, 2L, 3L, 4L, 5L}; - final var filter = XorBinaryFuse16.construct(keys); + final var filter = constructor.apply(keys); final var smallBuffer = ByteBuffer.allocate(filter.getSerializedSize() - 1); // Act & Assert @@ -150,7 +182,7 @@ public void shouldThrowExceptionWhenDeserializeBufferTooSmall() { // Act & Assert try { - XorBinaryFuse16.deserialize(tooSmallBuffer); + deserializer.apply(tooSmallBuffer); fail("Should have thrown IllegalArgumentException for buffer too small"); } catch (IllegalArgumentException e) { assertEquals("Buffer too small", e.getMessage()); @@ -165,20 +197,20 @@ public void shouldHandleFilterWithSequentialKeys() { for (int i = 0; i < size; i++) { keys[i] = i; } - final var originalFilter = XorBinaryFuse16.construct(keys); + final var originalFilter = constructor.apply(keys); final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); // Act originalFilter.serialize(buffer); buffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + final var deserializedFilter = deserializer.apply(buffer); // Assert for (int i = 0; i < size; i++) { - assertTrue("Sequential key " + i + " should be present", + assertTrue("Sequential key " + i + " should be present in " + filterName, deserializedFilter.mayContain(i)); } - assertFalse("Key outside range should not be in filter", + assertFalse("Key outside range should not be in " + filterName + " filter", deserializedFilter.mayContain(size + 1000)); } @@ -192,17 +224,17 @@ public void shouldHandleFilterWithRandomLargeKeys() { Long.MAX_VALUE / 2, Long.MAX_VALUE / 3 }; - final var originalFilter = XorBinaryFuse16.construct(keys); + final var originalFilter = constructor.apply(keys); final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); // Act originalFilter.serialize(buffer); buffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(buffer); + final var deserializedFilter = deserializer.apply(buffer); // Assert for (final long key : keys) { - assertTrue("Large key " + key + " should be present", + assertTrue("Large key " + key + " should be present in " + filterName, deserializedFilter.mayContain(key)); } } @@ -211,7 +243,7 @@ public void shouldHandleFilterWithRandomLargeKeys() { public void shouldCorrectlyCalculateSerializedSize() { // Arrange final var keys = new long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L}; - final var filter = XorBinaryFuse16.construct(keys); + final var filter = constructor.apply(keys); final int expectedSizeInBytes = filter.getSerializedSize(); final var buffer = ByteBuffer.allocate(expectedSizeInBytes); @@ -219,9 +251,9 @@ public void shouldCorrectlyCalculateSerializedSize() { filter.serialize(buffer); // Assert - assertEquals("Buffer position should equal serialized size", + assertEquals("Buffer position should equal serialized size for " + filterName, expectedSizeInBytes, buffer.position()); - assertEquals("Buffer should have no remaining space", + assertEquals("Buffer should have no remaining space for " + filterName, 0, buffer.remaining()); } @@ -229,37 +261,37 @@ public void shouldCorrectlyCalculateSerializedSize() { public void shouldHandleExactBufferSize() { // Arrange final var keys = new long[]{100L, 200L, 300L}; - final var filter = XorBinaryFuse16.construct(keys); + final var filter = constructor.apply(keys); final var exactBuffer = ByteBuffer.allocate(filter.getSerializedSize()); // Act filter.serialize(exactBuffer); exactBuffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(exactBuffer); + final var deserializedFilter = deserializer.apply(exactBuffer); // Assert for (final long key : keys) { - assertTrue("Key " + key + " should be present with exact buffer", + assertTrue("Key " + key + " should be present with exact buffer in " + filterName, deserializedFilter.mayContain(key)); } - assertEquals("No bytes should remain in buffer", 0, exactBuffer.remaining()); + assertEquals("No bytes should remain in buffer for " + filterName, 0, exactBuffer.remaining()); } @Test public void shouldHandleLargerBufferThanNeeded() { // Arrange final var keys = new long[]{1L, 2L, 3L}; - final var filter = XorBinaryFuse16.construct(keys); + final var filter = constructor.apply(keys); final var largeBuffer = ByteBuffer.allocate(filter.getSerializedSize() + 1000); // Act filter.serialize(largeBuffer); largeBuffer.flip(); - final var deserializedFilter = XorBinaryFuse16.deserialize(largeBuffer); + final var deserializedFilter = deserializer.apply(largeBuffer); // Assert for (final long key : keys) { - assertTrue("Key " + key + " should be present with larger buffer", + assertTrue("Key " + key + " should be present with larger buffer in " + filterName, deserializedFilter.mayContain(key)); } } From 97f874943d8339dcbf173f133771b8b6979b7c42 Mon Sep 17 00:00:00 2001 From: Vladimir Prus Date: Mon, 19 Jan 2026 19:13:22 +0000 Subject: [PATCH 24/38] XorBinaryFuse: throw on failure Filling the fingerprints array with 0xFF would create a filter that generates false negatives. --- .../main/java/org/fastfilter/xor/XorBinaryFuse16.java | 9 +++------ .../main/java/org/fastfilter/xor/XorBinaryFuse32.java | 9 +++------ .../src/main/java/org/fastfilter/xor/XorBinaryFuse8.java | 9 +++------ 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index aad3c77..0f497db 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -200,12 +200,9 @@ private void addAll(long[] keys) { if (hashIndex > 100) { // if construction doesn't succeed eventually, - // then there is likely a problem with the hash function - // let us not crash the system: - for(int i = 0; i < fingerprints.length; i++) { - fingerprints[i] = (short)0xFFFF; - } - return; + // then there is likely a problem with the hash function. + // It's better fail that either produce non-functional or incorrect filter. + throw new IllegalArgumentException("could not construct filter"); } // use a new random numbers seed = Hash.randomSeed(); diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java index d3f6125..67900a6 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java @@ -200,12 +200,9 @@ private void addAll(long[] keys) { if (hashIndex > 100) { // if construction doesn't succeed eventually, - // then there is likely a problem with the hash function - // let us not crash the system: - for (int i = 0; i < fingerprints.length; i++) { - fingerprints[i] = (int) 0xFFFFFFFF; - } - return; + // then there is likely a problem with the hash function. + // It's better fail that either produce non-functional or incorrect filter. + throw new IllegalArgumentException("could not construct filter"); } // use a new random numbers seed = Hash.randomSeed(); diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java index dfd5f45..81e42f1 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java @@ -200,12 +200,9 @@ private void addAll(long[] keys) { if (hashIndex > 100) { // if construction doesn't succeed eventually, - // then there is likely a problem with the hash function - // let us not crash the system: - for(int i = 0; i < fingerprints.length; i++) { - fingerprints[i] = (byte)0xFF; - } - return; + // then there is likely a problem with the hash function. + // It's better fail that either produce non-functional or incorrect filter. + throw new IllegalArgumentException("could not construct filter"); } // use a new random numbers seed = Hash.randomSeed(); From 5d2b41d4c9161035baa9236452ac84c5186a6553 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 19 Jan 2026 16:20:40 -0500 Subject: [PATCH 25/38] saving. --- README.md | 56 ++++++++ .../org/fastfilter/xor/StringFilters.java | 131 ++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 fastfilter/src/test/java/org/fastfilter/xor/StringFilters.java diff --git a/README.md b/README.md index 50ca0fd..48809d0 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,62 @@ The following additional types are implemented, but less tested: ## Usage + +To use the XOR and Binary Fuse filters, first prepare an array of keys, then construct the filter: + +```java +import org.fastfilter.xor.Xor8; +import org.fastfilter.xor.Xor16; +import org.fastfilter.xor.XorBinaryFuse8; +import org.fastfilter.xor.XorBinaryFuse16; + +// Example keys +long[] keys = {1, 2, 3, 4, 5}; + +// Construct XOR filters +Xor8 xor8 = Xor8.construct(keys); +Xor16 xor16 = Xor16.construct(keys); +XorBinaryFuse8 xorBinaryFuse8 = XorBinaryFuse8.construct(keys); +XorBinaryFuse16 xorBinaryFuse16 = XorBinaryFuse16.construct(keys); + +// Check membership +boolean mightContain = xor8.mayContain(1L); // true +boolean mightContain2 = xor8.mayContain(6L); // false (with high probability) +``` + +All filters implement the `Filter` interface and support the `mayContain(long key)` method to check if a key might be in the set. Note that false positives are possible, but false negatives are not. + + +### Serialization and Deserialization + +Filters can be serialized to and deserialized from a `ByteBuffer` for persistence or transmission: + +```java +import java.nio.ByteBuffer; + +// Assuming you have a constructed filter, e.g., Xor8 xor8 = Xor8.construct(keys); + +// Get the serialized size +int size = xor8.getSerializedSize(); + +// Allocate a ByteBuffer +ByteBuffer buffer = ByteBuffer.allocate(size); + +// Serialize the filter +xor8.serialize(buffer); + +// Prepare buffer for reading (flip) +buffer.flip(); + +// Deserialize the filter +Xor8 deserializedXor8 = Xor8.deserialize(buffer); + +// The deserialized filter behaves identically to the original +boolean result = deserializedXor8.mayContain(1L); // true +``` + +This allows saving filters to files, databases, or sending them over networks. + ### Maven When using Maven: The latest version, 1.0.4, is not yet available on Maven central, see [issue #48](https://github.com/FastFilter/fastfilter_java/issues/48). However, it is available at https://jitpack.io/: diff --git a/fastfilter/src/test/java/org/fastfilter/xor/StringFilters.java b/fastfilter/src/test/java/org/fastfilter/xor/StringFilters.java new file mode 100644 index 0000000..35f7324 --- /dev/null +++ b/fastfilter/src/test/java/org/fastfilter/xor/StringFilters.java @@ -0,0 +1,131 @@ +package org.fastfilter.xor; + +import static org.junit.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Random; +import java.util.Set; + +import org.fastfilter.Filter; +import org.junit.Test; + +public class StringFilters { + + private static final int NUM_STRINGS = 100_000; + private static final int NUM_TEST_STRINGS = 1_000; + private static final Random random = new Random(42); + + private static final long[] keys = generateKeys(); + private static final long[] testKeys = generateTestKeys(); + + private static long[] generateKeys() { + String[] strings = new String[NUM_STRINGS]; + for (int i = 0; i < NUM_STRINGS; i++) { + strings[i] = generateRandomString(); + } + long[] k = new long[NUM_STRINGS]; + for (int i = 0; i < NUM_STRINGS; i++) { + k[i] = hashString(strings[i]); + } + checkUniqueness(k, "keys"); + return k; + } + + private static long[] generateTestKeys() { + String[] strings = new String[NUM_TEST_STRINGS]; + for (int i = 0; i < NUM_TEST_STRINGS; i++) { + strings[i] = generateRandomString(); + } + long[] k = new long[NUM_TEST_STRINGS]; + for (int i = 0; i < NUM_TEST_STRINGS; i++) { + k[i] = hashString(strings[i]); + } + checkUniqueness(k, "test keys"); + return k; + } + + private static void checkUniqueness(long[] array, String name) { + Set set = new HashSet<>(); + int collisions = 0; + for (long l : array) { + if (!set.add(l)) { + collisions++; + } + } + if (collisions > 0) { + System.out.println("Warning: " + collisions + " hash collisions in " + name); + } else { + System.out.println("No hash collisions in " + name); + } + } + + private static String generateRandomString() { + int length = 5 + random.nextInt(16); // 5 to 20 chars + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + sb.append((char) ('a' + random.nextInt(26))); + } + return sb.toString(); + } + + private static long hashString(String s) { + long h = 0; + for (char c : s.toCharArray()) { + h = h * 31 + c; + } + return h; + } + + @Test + public void testXor8() { + testFilter(Xor8.class); + } + + @Test + public void testXor16() { + testFilter(Xor16.class); + } + + @Test + public void testXorBinaryFuse8() { + testFilter(XorBinaryFuse8.class); + } + + @Test + public void testXorBinaryFuse16() { + testFilter(XorBinaryFuse16.class); + } + + @Test + public void testXorBinaryFuse32() { + testFilter(XorBinaryFuse32.class); + } + + private void testFilter(Class filterClass) { + // Construct filter + Filter filter; + try { + filter = (Filter) filterClass.getMethod("construct", long[].class).invoke(null, (Object) keys); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // Check all keys are in the filter + for (int i = 0; i < NUM_STRINGS; i++) { + assertTrue("Key " + i + " should be in filter", filter.mayContain(keys[i])); + } + + // Check false positives on test keys + int falsePositives = 0; + for (int i = 0; i < NUM_TEST_STRINGS; i++) { + if (filter.mayContain(testKeys[i])) { + falsePositives++; + } + } + + // Expect low false positive rate (less than 1% for most filters) + double fpp = (double) falsePositives / NUM_TEST_STRINGS; + System.out.println(filterClass.getSimpleName() + " false positive rate: " + fpp); + assertTrue("False positive rate should be low: " + fpp, fpp < 0.01); // Allow up to 1% + } +} From 55d5510107f02a90ee8fecbb610dca2b03ecfa19 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 19 Jan 2026 18:58:15 -0500 Subject: [PATCH 26/38] This makes XorBinaryFuse16 and XorBinaryFuse8 more robust --- README.md | 22 ++-- .../main/java/org/fastfilter/xor/Xor16.java | 3 + .../main/java/org/fastfilter/xor/Xor8.java | 4 + .../org/fastfilter/xor/XorBinaryFuse16.java | 105 ++++++++++-------- .../org/fastfilter/xor/XorBinaryFuse32.java | 3 +- .../org/fastfilter/xor/XorBinaryFuse8.java | 105 ++++++++++-------- 6 files changed, 138 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 48809d0..3437b8a 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The following additional types are implemented, but less tested: ## Reference -* Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 +* Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 * Thomas Mueller Graf, Daniel Lemire, [Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters](https://arxiv.org/abs/1912.08258), Journal of Experimental Algorithmics 25 (1), 2020. DOI: 10.1145/3376122 ## Usage @@ -31,17 +31,13 @@ The following additional types are implemented, but less tested: To use the XOR and Binary Fuse filters, first prepare an array of keys, then construct the filter: ```java -import org.fastfilter.xor.Xor8; -import org.fastfilter.xor.Xor16; import org.fastfilter.xor.XorBinaryFuse8; import org.fastfilter.xor.XorBinaryFuse16; // Example keys long[] keys = {1, 2, 3, 4, 5}; -// Construct XOR filters -Xor8 xor8 = Xor8.construct(keys); -Xor16 xor16 = Xor16.construct(keys); +// Construct binary fuse filters= XorBinaryFuse8 xorBinaryFuse8 = XorBinaryFuse8.construct(keys); XorBinaryFuse16 xorBinaryFuse16 = XorBinaryFuse16.construct(keys); @@ -52,6 +48,11 @@ boolean mightContain2 = xor8.mayContain(6L); // false (with high probability) All filters implement the `Filter` interface and support the `mayContain(long key)` method to check if a key might be in the set. Note that false positives are possible, but false negatives are not. +### Generating the Hash Values + +The library is written to process `long` values that are meant to be hash values. Though you do not need to use +cryptographically strong hashing, you should make sure that your hash functions are reasonable: they should +not generate too many collisions (two objects mapping to the same `long` value). ### Serialization and Deserialization @@ -60,25 +61,24 @@ Filters can be serialized to and deserialized from a `ByteBuffer` for persistenc ```java import java.nio.ByteBuffer; -// Assuming you have a constructed filter, e.g., Xor8 xor8 = Xor8.construct(keys); +// Assuming you have a constructed filter // Get the serialized size -int size = xor8.getSerializedSize(); +int size = XorBinaryFuse8.getSerializedSize(); // Allocate a ByteBuffer ByteBuffer buffer = ByteBuffer.allocate(size); // Serialize the filter -xor8.serialize(buffer); +XorBinaryFuse8.serialize(buffer); // Prepare buffer for reading (flip) buffer.flip(); // Deserialize the filter -Xor8 deserializedXor8 = Xor8.deserialize(buffer); +XorBinaryFuse8 deserializedXorBinaryFuse8 = Xor8.deserialize(buffer); // The deserialized filter behaves identically to the original -boolean result = deserializedXor8.mayContain(1L); // true ``` This allows saving filters to files, databases, or sending them over networks. diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java index 8cdc4d8..603a3f1 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java @@ -6,7 +6,10 @@ import org.fastfilter.utils.Hash; /** + * The Xor16 filter implementation is experimental. We recommend using XorBinaryFuse16 instead. Use at your own risks. + * * The xor filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters](https://arxiv.org/abs/1912.08258), Journal of Experimental Algorithmics 25 (1), 2020. DOI: 10.1145/3376122 * * It needs 1.23 log(1/fpp) bits per key. It is related to the BDZ algorithm [1] * (a minimal perfect hash function algorithm). diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java index bb3b5ff..a54b0d9 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java @@ -6,8 +6,12 @@ import org.fastfilter.Filter; import org.fastfilter.utils.Hash; + /** + * The Xor8 filter implementation is experimental. We recommend using XorBinaryFuse8 instead. Use at your own risks. + * * The xor filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters](https://arxiv.org/abs/1912.08258), Journal of Experimental Algorithmics 25 (1), 2020. DOI: 10.1145/3376122 * * It needs 1.23 log(1/fpp) bits per key. It is related to the BDZ algorithm [1] * (a minimal perfect hash function algorithm). diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index e7332fd..b0ef76a 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -7,6 +7,7 @@ /** * The xor binary fuse filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 */ public class XorBinaryFuse16 implements Filter { @@ -78,6 +79,15 @@ private static int mod3(int x) { return x; } + /** + * Constructs a new XorBinaryFuse16 filter from the given array of keys. + * The filter is designed to have a low false positive rate while being space-efficient. + * The keys array should contain unique values. The array may be mutated during construction + * (e.g., sorted and deduplicated) if the algorithm detects that there are likely too many duplicates. + * + * @param keys the array of long keys to add to the filter + * @return a new XorBinaryFuse16 filter containing all the keys + */ public static XorBinaryFuse16 construct(long[] keys) { int size = keys.length; int segmentLength = calculateSegmentLength(ARITY, size); @@ -102,6 +112,7 @@ private void addAll(long[] keys) { long[] reverseOrder = new long[size + 1]; byte[] reverseH = new byte[size]; int reverseOrderPos = 0; + boolean duplicated = false; // the lowest 2 bits are the h index (0, 1, or 2) // so we only have 6 bits for counting; @@ -117,7 +128,6 @@ private void addAll(long[] keys) { blockBits++; } int block = 1 << blockBits; - mainloop: while (true) { reverseOrder[size] = 1; int[] startPos = new int[block]; @@ -126,7 +136,8 @@ private void addAll(long[] keys) { } // counting sort - for (long key : keys) { + for(int i = 0; i < size; i++) { + long key = keys[i]; long hash = Hash.hash64(key, seed); int segmentIndex = (int) (hash >>> (64 - blockBits)); // We only overwrite when the hash was zero. Zero hash values @@ -150,52 +161,48 @@ private void addAll(long[] keys) { } } startPos = null; - if (countMask < 0) { - // we have a possible counter overflow - continue mainloop; - } - - reverseOrderPos = 0; - int alonePos = 0; - for (int i = 0; i < arrayLength; i++) { - alone[alonePos] = i; - int inc = (t2count[i] >> 2) == 1 ? 1 : 0; - alonePos += inc; - } + if (countMask >= 0) { + reverseOrderPos = 0; + int alonePos = 0; + for (int i = 0; i < arrayLength; i++) { + alone[alonePos] = i; + int inc = (t2count[i] >> 2) == 1 ? 1 : 0; + alonePos += inc; + } - while (alonePos > 0) { - alonePos--; - int index = alone[alonePos]; - if ((t2count[index] >> 2) == 1) { - // It is still there! - long hash = t2hash[index]; - byte found = (byte) (t2count[index] & 3); - - reverseH[reverseOrderPos] = found; - reverseOrder[reverseOrderPos] = hash; - - h012[0] = getHashFromHash(hash, 0); - h012[1] = getHashFromHash(hash, 1); - h012[2] = getHashFromHash(hash, 2); - - int index3 = h012[mod3(found + 1)]; - alone[alonePos] = index3; - alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); - t2count[index3] -= 4; - t2count[index3] ^= mod3(found + 1); - t2hash[index3] ^= hash; - - index3 = h012[mod3(found + 2)]; - alone[alonePos] = index3; - alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); - t2count[index3] -= 4; - t2count[index3] ^= mod3(found + 2); - t2hash[index3] ^= hash; - - reverseOrderPos++; + while (alonePos > 0) { + alonePos--; + int index = alone[alonePos]; + if ((t2count[index] >> 2) == 1) { + // It is still there! + long hash = t2hash[index]; + byte found = (byte) (t2count[index] & 3); + + reverseH[reverseOrderPos] = found; + reverseOrder[reverseOrderPos] = hash; + + h012[0] = getHashFromHash(hash, 0); + h012[1] = getHashFromHash(hash, 1); + h012[2] = getHashFromHash(hash, 2); + + int index3 = h012[mod3(found + 1)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 1); + t2hash[index3] ^= hash; + + index3 = h012[mod3(found + 2)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 2); + t2hash[index3] ^= hash; + + reverseOrderPos++; + } } } - if (reverseOrderPos == size) { break; } @@ -203,7 +210,13 @@ private void addAll(long[] keys) { Arrays.fill(t2count, (byte) 0); Arrays.fill(t2hash, 0); Arrays.fill(reverseOrder, 0); - + // If we reach 10 passes, we assume that there are too many duplicates + // in the input key set. We then sort and remove duplicates in place. + // This should almost never happen. + if (countMask < 0 && !duplicated) { + size = Deduplicator.sortAndRemoveDup(keys, size); + duplicated = true; + } if (hashIndex > 100) { // if construction doesn't succeed eventually, // then there is likely a problem with the hash function. diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java index 29df2e5..fc1a01b 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java @@ -7,7 +7,8 @@ import org.fastfilter.utils.Hash; /** - * The xor binary fuse filter, a new algorithm that can replace a Bloom filter. + * The XorBinaryFuse32 filter is experimental. We recommend using XorBinaryFuse8 or XorBinaryFuse16 instead. + * Use at your own risks. */ public class XorBinaryFuse32 implements Filter { diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java index 29b1618..e8f0337 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java @@ -8,6 +8,7 @@ /** * The xor binary fuse filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 */ public class XorBinaryFuse8 implements Filter { @@ -79,6 +80,15 @@ private static int mod3(int x) { return x; } + /** + * Constructs a new XorBinaryFuse8 filter from the given array of keys. + * The filter is designed to have a low false positive rate while being space-efficient. + * The keys array should contain unique values. The array may be mutated during construction + * (e.g., sorted and deduplicated) if the algorithm detects that there are likely too many duplicates. + * + * @param keys the array of long keys to add to the filter + * @return a new XorBinaryFuse8 filter containing all the keys + */ public static XorBinaryFuse8 construct(long[] keys) { int size = keys.length; int segmentLength = calculateSegmentLength(ARITY, size); @@ -103,6 +113,7 @@ private void addAll(long[] keys) { long[] reverseOrder = new long[size + 1]; byte[] reverseH = new byte[size]; int reverseOrderPos = 0; + boolean duplicated = false; // the lowest 2 bits are the h index (0, 1, or 2) // so we only have 6 bits for counting; @@ -118,7 +129,6 @@ private void addAll(long[] keys) { blockBits++; } int block = 1 << blockBits; - mainloop: while (true) { reverseOrder[size] = 1; int[] startPos = new int[block]; @@ -126,8 +136,8 @@ private void addAll(long[] keys) { startPos[i] = (int) ((long) i * size / block); } // counting sort - - for (long key : keys) { + for(int i = 0; i < size; i++) { + long key = keys[i]; long hash = Hash.hash64(key, seed); int segmentIndex = (int) (hash >>> (64 - blockBits)); // We only overwrite when the hash was zero. Zero hash values @@ -151,49 +161,46 @@ private void addAll(long[] keys) { } } startPos = null; - if (countMask < 0) { - // we have a possible counter overflow - continue mainloop; - } - - reverseOrderPos = 0; - int alonePos = 0; - for (int i = 0; i < arrayLength; i++) { - alone[alonePos] = i; - int inc = (t2count[i] >> 2) == 1 ? 1 : 0; - alonePos += inc; - } + if (countMask >= 0) { + reverseOrderPos = 0; + int alonePos = 0; + for (int i = 0; i < arrayLength; i++) { + alone[alonePos] = i; + int inc = (t2count[i] >> 2) == 1 ? 1 : 0; + alonePos += inc; + } - while (alonePos > 0) { - alonePos--; - int index = alone[alonePos]; - if ((t2count[index] >> 2) == 1) { - // It is still there! - long hash = t2hash[index]; - byte found = (byte) (t2count[index] & 3); - - reverseH[reverseOrderPos] = found; - reverseOrder[reverseOrderPos] = hash; - - h012[0] = getHashFromHash(hash, 0); - h012[1] = getHashFromHash(hash, 1); - h012[2] = getHashFromHash(hash, 2); - - int index3 = h012[mod3(found + 1)]; - alone[alonePos] = index3; - alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); - t2count[index3] -= 4; - t2count[index3] ^= mod3(found + 1); - t2hash[index3] ^= hash; - - index3 = h012[mod3(found + 2)]; - alone[alonePos] = index3; - alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); - t2count[index3] -= 4; - t2count[index3] ^= mod3(found + 2); - t2hash[index3] ^= hash; - - reverseOrderPos++; + while (alonePos > 0) { + alonePos--; + int index = alone[alonePos]; + if ((t2count[index] >> 2) == 1) { + // It is still there! + long hash = t2hash[index]; + byte found = (byte) (t2count[index] & 3); + + reverseH[reverseOrderPos] = found; + reverseOrder[reverseOrderPos] = hash; + + h012[0] = getHashFromHash(hash, 0); + h012[1] = getHashFromHash(hash, 1); + h012[2] = getHashFromHash(hash, 2); + + int index3 = h012[mod3(found + 1)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 1); + t2hash[index3] ^= hash; + + index3 = h012[mod3(found + 2)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 2); + t2hash[index3] ^= hash; + + reverseOrderPos++; + } } } @@ -204,7 +211,13 @@ private void addAll(long[] keys) { Arrays.fill(t2count, (byte) 0); Arrays.fill(t2hash, 0); Arrays.fill(reverseOrder, 0); - + // If we reach 10 passes, we assume that there are too many duplicates + // in the input key set. We then sort and remove duplicates in place. + // This should almost never happen. + if (countMask < 0 && !duplicated) { + size = Deduplicator.sortAndRemoveDup(keys, size); + duplicated = true; + } if (hashIndex > 100) { // if construction doesn't succeed eventually, // then there is likely a problem with the hash function. From ff1e4b29f5c7fb9cdf70375967abd8e75e68832a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 19 Jan 2026 19:01:31 -0500 Subject: [PATCH 27/38] adding missing file --- .../java/org/fastfilter/xor/Deduplicator.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 fastfilter/src/main/java/org/fastfilter/xor/Deduplicator.java diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Deduplicator.java b/fastfilter/src/main/java/org/fastfilter/xor/Deduplicator.java new file mode 100644 index 0000000..6bd1759 --- /dev/null +++ b/fastfilter/src/main/java/org/fastfilter/xor/Deduplicator.java @@ -0,0 +1,27 @@ +package org.fastfilter.xor; + +import java.util.Arrays; + +public class Deduplicator { + + /** + * Sorts the keys array and removes duplicates in place. + * Returns the new length of the array (number of unique elements). + * + * @param keys the array of keys to deduplicate + * @param length the current length of the array + * @return the new length after removing duplicates + */ + public static int sortAndRemoveDup(long[] keys, int length) { + Arrays.sort(keys, 0, length); + int j = 1; + for (int i = 1; i < length; i++) { + if (keys[i] != keys[i - 1]) { + keys[j] = keys[i]; + j++; + } + } + return j; + } + +} From d5e2f6350b5a34b55f385afe3b88aff5b4bb3e6b Mon Sep 17 00:00:00 2001 From: Vladimir Prus Date: Tue, 20 Jan 2026 18:48:54 +0000 Subject: [PATCH 28/38] Xor8: remove previous serialization methods --- .../main/java/org/fastfilter/xor/Xor8.java | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java index a54b0d9..6faa557 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java @@ -1,6 +1,5 @@ package org.fastfilter.xor; -import java.io.*; import java.nio.ByteBuffer; import org.fastfilter.Filter; @@ -164,34 +163,6 @@ private int fingerprint(long hash) { return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1)); } - public byte[] getData() { - try { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - DataOutputStream d = new DataOutputStream(out); - d.writeInt(size); - d.writeLong(seed); - d.write(fingerprints); - return out.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public Xor8(InputStream in) { - try { - DataInputStream din = new DataInputStream(in); - size = din.readInt(); - arrayLength = getArrayLength(size); - bitCount = arrayLength * BITS_PER_FINGERPRINT; - blockLength = arrayLength / HASHES; - seed = din.readLong(); - fingerprints = new byte[arrayLength]; - din.readFully(fingerprints); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - private Xor8(int size, long seed, byte[] fingerprints) { this.size = size; this.arrayLength = getArrayLength(size); From 9e19bc61cabfe1cd2b15af27aec943f15b3a4428 Mon Sep 17 00:00:00 2001 From: Vladimir Prus Date: Tue, 20 Jan 2026 19:06:05 +0000 Subject: [PATCH 29/38] Xor8: make methods deprecated --- .../main/java/org/fastfilter/xor/Xor8.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java index 6faa557..142fb87 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java @@ -1,5 +1,6 @@ package org.fastfilter.xor; +import java.io.*; import java.nio.ByteBuffer; import org.fastfilter.Filter; @@ -163,6 +164,36 @@ private int fingerprint(long hash) { return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1)); } + @Deprecated + public byte[] getData() { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DataOutputStream d = new DataOutputStream(out); + d.writeInt(size); + d.writeLong(seed); + d.write(fingerprints); + return out.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Deprecated + public Xor8(InputStream in) { + try { + DataInputStream din = new DataInputStream(in); + size = din.readInt(); + arrayLength = getArrayLength(size); + bitCount = arrayLength * BITS_PER_FINGERPRINT; + blockLength = arrayLength / HASHES; + seed = din.readLong(); + fingerprints = new byte[arrayLength]; + din.readFully(fingerprints); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private Xor8(int size, long seed, byte[] fingerprints) { this.size = size; this.arrayLength = getArrayLength(size); From 42aef913ba9a3d56a08cf83f8f5742b27ed52906 Mon Sep 17 00:00:00 2001 From: Vladimir Prus Date: Wed, 21 Jan 2026 10:32:45 +0000 Subject: [PATCH 30/38] Xor: implement stream serialization There are already methods operating on ByteBuffer, however versions taking streams allow to avoid data copying, and more directly compatible with various network APIs. Also, for reference, Guava Bloom filter implementation uses streams. There's a bit of code duplication as the result. Sadly, there's no standard class to wrap ByteBuffer in a stream, and pulling in an external dependency seems like an overkill here. --- .../src/main/java/org/fastfilter/Filter.java | 13 ++++ .../main/java/org/fastfilter/xor/Xor16.java | 28 ++++++++ .../main/java/org/fastfilter/xor/Xor8.java | 18 +++++ .../org/fastfilter/xor/XorBinaryFuse16.java | 30 ++++++++ .../org/fastfilter/xor/XorBinaryFuse32.java | 30 ++++++++ .../org/fastfilter/xor/XorBinaryFuse8.java | 26 +++++++ .../org/fastfilter/xor/SerializationTest.java | 70 +++++++++++++++++-- 7 files changed, 209 insertions(+), 6 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/Filter.java b/fastfilter/src/main/java/org/fastfilter/Filter.java index 05bfe43..83bd4f6 100644 --- a/fastfilter/src/main/java/org/fastfilter/Filter.java +++ b/fastfilter/src/main/java/org/fastfilter/Filter.java @@ -1,5 +1,7 @@ package org.fastfilter; +import java.io.IOException; +import java.io.OutputStream; import java.nio.ByteBuffer; /** @@ -85,4 +87,15 @@ default int getSerializedSize() { default void serialize(ByteBuffer buffer) { throw new UnsupportedOperationException(); } + + /** + * Serializes the filter state into the provided {@code OutputStream}. + * + * @param out the output stream where the serialized state of the filter will be written + * @throws IOException if writing to the stream fails + * @throws UnsupportedOperationException if the operation is not supported by the filter implementation + */ + default void serialize(OutputStream out) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java index 603a3f1..265ec12 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import org.fastfilter.Filter; @@ -199,4 +204,27 @@ public static Xor16 deserialize(ByteBuffer buffer) { return new Xor16(blockLength, bitCount, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(blockLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final short fp : fingerprints) { + dout.writeShort(fp); + } + } + + public static Xor16 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int blockLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readShort(); + } + final int bitCount = len * BITS_PER_FINGERPRINT; + return new Xor16(blockLength, bitCount, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java index 142fb87..4b7c5b6 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java @@ -241,4 +241,22 @@ public static Xor8 deserialize(ByteBuffer buffer) { return new Xor8(size, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(size); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + dout.write(fingerprints); + } + + public static Xor8 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int size = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] fingerprints = new byte[len]; + din.readFully(fingerprints); + return new Xor8(size, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index b0ef76a..588e70c 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; import org.fastfilter.Filter; @@ -325,4 +330,29 @@ public static XorBinaryFuse16 deserialize(ByteBuffer buffer) { return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final short fp : fingerprints) { + dout.writeShort(fp); + } + } + + public static XorBinaryFuse16 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readShort(); + } + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java index fc1a01b..d226081 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -313,4 +318,29 @@ public static XorBinaryFuse32 deserialize(ByteBuffer buffer) { return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final int fp : fingerprints) { + dout.writeInt(fp); + } + } + + public static XorBinaryFuse32 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final int[] fingerprints = new int[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readInt(); + } + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java index e8f0337..f4e31ff 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -321,4 +326,25 @@ public static XorBinaryFuse8 deserialize(ByteBuffer buffer) { return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + dout.write(fingerprints); + } + + public static XorBinaryFuse8 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] fingerprints = new byte[len]; + din.readFully(fingerprints); + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java index df6a204..023475d 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -5,6 +5,10 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; import java.util.function.Function; @@ -20,28 +24,36 @@ public class SerializationTest { private final String filterName; private final Function constructor; private final Function deserializer; + private final StreamDeserializer streamDeserializer; public SerializationTest(String filterName, Function constructor, - Function deserializer) { + Function deserializer, + StreamDeserializer streamDeserializer) { this.filterName = filterName; this.constructor = constructor; this.deserializer = deserializer; + this.streamDeserializer = streamDeserializer; } @Parameters(name = "{0}") public static List filters() { return List.of( new Object[] {"Xor8", (Function) Xor8::construct, - (Function) Xor8::deserialize}, + (Function) Xor8::deserialize, + (StreamDeserializer) Xor8::deserialize}, new Object[] {"Xor16", (Function) Xor16::construct, - (Function) Xor16::deserialize}, + (Function) Xor16::deserialize, + (StreamDeserializer) Xor16::deserialize}, new Object[] {"XorBinaryFuse8", (Function) XorBinaryFuse8::construct, - (Function) XorBinaryFuse8::deserialize}, + (Function) XorBinaryFuse8::deserialize, + (StreamDeserializer) XorBinaryFuse8::deserialize}, new Object[] {"XorBinaryFuse16", (Function) XorBinaryFuse16::construct, - (Function) XorBinaryFuse16::deserialize}, + (Function) XorBinaryFuse16::deserialize, + (StreamDeserializer) XorBinaryFuse16::deserialize}, new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, - (Function) XorBinaryFuse32::deserialize} + (Function) XorBinaryFuse32::deserialize, + (StreamDeserializer) XorBinaryFuse32::deserialize} ); } @@ -85,6 +97,52 @@ public void shouldSerializeAndDeserializeMediumFilter() { assertFalse("Key 1500L should not be in " + filterName + " filter", deserializedFilter.mayContain(1500L)); } + @Test + public void shouldSerializeAndDeserializeMediumFilterFromStream() throws IOException { + // Arrange + final var keys = new long[]{100L, 200L, 300L, 400L, 500L, 600L, 700L, 800L, 900L, 1000L}; + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + final var input = new ByteArrayInputStream(buffer.array()); + final var deserializedFilter = streamDeserializer.deserialize(input); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 50L should not be in " + filterName + " filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 1500L should not be in " + filterName + " filter", deserializedFilter.mayContain(1500L)); + } + + @Test + public void shouldSerializeToStreamAndDeserializeFromByteBuffer() throws IOException { + // Arrange + final var keys = new long[]{10L, 20L, 30L, 40L, 50L, 60L, 70L, 80L}; + final var originalFilter = constructor.apply(keys); + final var out = new ByteArrayOutputStream(); + + // Act + originalFilter.serialize(out); + final var buffer = ByteBuffer.wrap(out.toByteArray()); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 15L should not be in " + filterName + " filter", deserializedFilter.mayContain(15L)); + } + + @FunctionalInterface + private interface StreamDeserializer { + Filter deserialize(InputStream in) throws IOException; + } + @Test public void shouldSerializeAndDeserializeLargeFilter() { // Arrange From cbc2090773ca60a96a71e574e80493704570cbcb Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 23 Feb 2026 08:37:42 +0100 Subject: [PATCH 31/38] [maven-release-plugin] prepare release fastfilter_java-1.0.5 --- fastfilter/pom.xml | 2 +- jmh/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index 2e84df2..98257fa 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5-SNAPSHOT + 1.0.5 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index 4be23ea..fdb01a1 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5-SNAPSHOT + 1.0.5 4.0.0 diff --git a/pom.xml b/pom.xml index fbf3342..8703efb 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.github.fastfilter fastfilter_java pom - 1.0.5-SNAPSHOT + 1.0.5 fastfilter jmh @@ -47,7 +47,7 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - HEAD + fastfilter_java-1.0.5 From 21e1714720e34d1f0a69580fe701486959238427 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 23 Feb 2026 08:37:46 +0100 Subject: [PATCH 32/38] [maven-release-plugin] prepare for next development iteration --- fastfilter/pom.xml | 2 +- jmh/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index 98257fa..5178f9f 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5 + 1.0.6-SNAPSHOT 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index fdb01a1..88db1ad 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5 + 1.0.6-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 8703efb..727c8f2 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.github.fastfilter fastfilter_java pom - 1.0.5 + 1.0.6-SNAPSHOT fastfilter jmh @@ -47,7 +47,7 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - fastfilter_java-1.0.5 + HEAD From e1d04aed852e7c551bdae696bced7a1e75b5a4ba Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 26 Feb 2026 09:12:25 +0100 Subject: [PATCH 33/38] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3437b8a..895dea1 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ When using Maven: The latest version, 1.0.4, is not yet available on Maven centr io.github.fastfilter fastfilter - 1.0.4 + 1.0.5 ### Gradle @@ -110,7 +110,7 @@ When using Maven: The latest version, 1.0.4, is not yet available on Maven centr } dependencies { - implementation 'io.github.fastfilter.fastfilter_java:fastfilter:1.0.4' + implementation 'io.github.fastfilter.fastfilter_java:fastfilter:1.0.5' } ### Maven Central (version 1.0.2) From 993bc9851d1988ae9766f6303ac6df36bdd888f3 Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Wed, 29 Jul 2026 22:27:52 +0530 Subject: [PATCH 34/38] Add RibbonFilter core: banding, back-substitution, and query Implements the standard (non-homogeneous, w=64) Ribbon filter construction and membership query described in issue #33, following Xor8's structure (hash-derive, build-with-retries, query-by-recombine). --- .../org/fastfilter/ribbon/RibbonFilter.java | 144 ++++++++++++++++++ .../fastfilter/ribbon/RibbonFilterTest.java | 52 +++++++ 2 files changed, 196 insertions(+) create mode 100644 fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java create mode 100644 fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java new file mode 100644 index 0000000..138fe3d --- /dev/null +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java @@ -0,0 +1,144 @@ +package org.fastfilter.ribbon; + +import org.fastfilter.Filter; +import org.fastfilter.utils.Hash; + +/** + * A (non-homogeneous) Ribbon filter with a fixed ribbon width w = 64. + * + * A Ribbon filter is a retrieval data structure: for every key x in the + * set, solving a banded linear system over GF(2) yields a small function + * f(x) that returns an r-bit fingerprint. Construction can fail (linear + * dependence among the equations) and is retried with a new seed, and - + * if all seed attempts at a given slot count fail - with more slots. + * + */ +public class RibbonFilter implements Filter { + + private static final int DEFAULT_RESULT_BITS = 8; + private static final int COEFF_BITS = 64; + private static final int MAX_ATTEMPTS_PER_SIZE = 256; + private static final double INITIAL_OVERHEAD_FACTOR = 0.10; + private static final double OVERHEAD_GROWTH_FACTOR = 1.5; + + private final int size; + private final int resultBits; + private final int numSlots; + private final int numStarts; + private final long seed; + private final byte[] solution; + private final int bitCount; + + private RibbonFilter(int size, int resultBits, int numSlots, int numStarts, long seed, byte[] solution) { + this.size = size; + this.resultBits = resultBits; + this.numSlots = numSlots; + this.numStarts = numStarts; + this.seed = seed; + this.solution = solution; + this.bitCount = numSlots * resultBits; + } + + public static RibbonFilter construct(long[] keys) { + return construct(keys, DEFAULT_RESULT_BITS); + } + + public static RibbonFilter construct(long[] keys, int resultBits) { + if (resultBits < 1 || resultBits > 8) { + throw new IllegalArgumentException("resultBits must be between 1 and 8"); + } + int size = keys.length; + double overheadFactor = INITIAL_OVERHEAD_FACTOR; + while (true) { + int numStarts = (int) Math.ceil(size * (1 + overheadFactor)); + if (numStarts < 1) { + numStarts = 1; + } + int numSlots = numStarts + COEFF_BITS - 1; + for (int attempt = 0; attempt < MAX_ATTEMPTS_PER_SIZE; attempt++) { + long seed = Hash.randomSeed(); + long[] coeff = new long[numSlots]; + byte[] result = new byte[numSlots]; + if (band(keys, seed, numStarts, resultBits, coeff, result)) { + byte[] solution = backSubstitute(coeff, result, numSlots, resultBits); + return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + } + } + overheadFactor *= OVERHEAD_GROWTH_FACTOR; + } + } + + private static boolean band(long[] keys, long seed, int numStarts, int resultBits, long[] coeff, byte[] result) { + int mask = (1 << resultBits) - 1; + for (long key : keys) { + long hash = Hash.hash64(key, seed); + int s = Hash.reduce((int) hash, numStarts); + long c = Long.rotateLeft(hash, 21) | 1L; + int r = (int) (Long.rotateLeft(hash, 42) & mask); + while (true) { + int i = Long.numberOfTrailingZeros(c); + int p = s + i; + c >>>= i; + if (coeff[p] == 0) { + coeff[p] = c; + result[p] = (byte) r; + break; + } + c ^= coeff[p]; + r ^= result[p]; + s = p; + if (c == 0) { + return false; + } + } + } + return true; + } + + private static byte[] backSubstitute(long[] coeff, byte[] result, int numSlots, int resultBits) { + byte[] solution = new byte[numSlots]; + int mask = (1 << resultBits) - 1; + for (int i = numSlots - 1; i >= 0; i--) { + long c = coeff[i]; + if (c == 0) { + continue; + } + int contribution = 0; + long rest = c & ~1L; + while (rest != 0) { + int k = Long.numberOfTrailingZeros(rest); + rest &= rest - 1; + contribution ^= solution[i + k]; + } + solution[i] = (byte) ((result[i] ^ contribution) & mask); + } + return solution; + } + + @Override + public boolean mayContain(long key) { + long hash = Hash.hash64(key, seed); + int s = Hash.reduce((int) hash, numStarts); + long c = Long.rotateLeft(hash, 21) | 1L; + int mask = (1 << resultBits) - 1; + int expected = (int) (Long.rotateLeft(hash, 42) & mask); + + int actual = 0; + for (int j = 0; j < resultBits; j++) { + long columnBits = 0; + for (int k = 0; k < COEFF_BITS; k++) { + if (((solution[s + k] >>> j) & 1) != 0) { + columnBits |= (1L << k); + } + } + int bit = Long.bitCount(columnBits & c) & 1; + actual |= bit << j; + } + return actual == expected; + } + + @Override + public long getBitCount() { + return bitCount; + } +} diff --git a/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java new file mode 100644 index 0000000..a150aba --- /dev/null +++ b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java @@ -0,0 +1,52 @@ +package org.fastfilter.ribbon; + +import static org.junit.Assert.assertTrue; + +import org.fastfilter.utils.RandomGenerator; +import org.junit.Test; + +public class RibbonFilterTest { + + @Test + public void noFalseNegativesSmall() { + assertNoFalseNegatives(10); + assertNoFalseNegatives(1_000); + } + + @Test + public void noFalseNegativesLarge() { + assertNoFalseNegatives(1_000_000); + } + + @Test + public void falsePositiveRateNearExpected() { + int len = 1_000_000; + long[] list = new long[len * 2]; + RandomGenerator.createRandomUniqueListFast(list, 0); + long[] keys = new long[len]; + long[] nonKeys = new long[len]; + for (int i = 0; i < len; i++) { + keys[i] = list[i]; + nonKeys[i] = list[i + len]; + } + RibbonFilter filter = RibbonFilter.construct(keys); + int falsePositives = 0; + for (long nonKey : nonKeys) { + if (filter.mayContain(nonKey)) { + falsePositives++; + } + } + double fpp = (double) falsePositives / len; + // expected ~= 1/256 ~= 0.0039; generous margin to avoid flakiness + assertTrue("fpp too high: " + fpp, fpp < 0.01); + } + + private void assertNoFalseNegatives(int len) { + long[] keys = new long[len]; + RandomGenerator.createRandomUniqueListFast(keys, 0); + RibbonFilter filter = RibbonFilter.construct(keys); + for (long key : keys) { + assertTrue("key " + key + " should be present", filter.mayContain(key)); + } + } +} From f774f0f161a111e3fc960f35f77bb1beeba419dc Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Wed, 29 Jul 2026 22:33:00 +0530 Subject: [PATCH 35/38] Add RibbonFilter serialization and round-trip test coverage Mirrors Xor8's ByteBuffer/OutputStream serialize + static deserialize convention. Unlike Xor8, numSlots/numStarts must be stored explicitly (not re-derived from size) since the construction retry loop can grow the slot count beyond the simple sizing formula. --- .../org/fastfilter/ribbon/RibbonFilter.java | 69 +++++++++++++++++++ .../org/fastfilter/xor/SerializationTest.java | 6 +- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java index 138fe3d..5ec9a8f 100644 --- a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java @@ -1,5 +1,12 @@ package org.fastfilter.ribbon; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + import org.fastfilter.Filter; import org.fastfilter.utils.Hash; @@ -141,4 +148,66 @@ public boolean mayContain(long key) { public long getBitCount() { return bitCount; } + + @Override + public int getSerializedSize() { + return Integer.BYTES * 5 + Long.BYTES + solution.length * Byte.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + buffer.putInt(size); + buffer.putInt(resultBits); + buffer.putInt(numSlots); + buffer.putInt(numStarts); + buffer.putLong(seed); + buffer.putInt(solution.length); + buffer.put(solution); + } + + public static RibbonFilter deserialize(ByteBuffer buffer) { + if (buffer.remaining() < Integer.BYTES * 5 + Long.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + final int size = buffer.getInt(); + final int resultBits = buffer.getInt(); + final int numSlots = buffer.getInt(); + final int numStarts = buffer.getInt(); + final long seed = buffer.getLong(); + final int len = buffer.getInt(); + if (buffer.remaining() < len * Byte.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + final byte[] solution = new byte[len]; + buffer.get(solution); + return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + } + + @Override + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(size); + dout.writeInt(resultBits); + dout.writeInt(numSlots); + dout.writeInt(numStarts); + dout.writeLong(seed); + dout.writeInt(solution.length); + dout.write(solution); + } + + public static RibbonFilter deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int size = din.readInt(); + final int resultBits = din.readInt(); + final int numSlots = din.readInt(); + final int numStarts = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] solution = new byte[len]; + din.readFully(solution); + return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + } } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java index 023475d..b2435ec 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.function.Function; import org.fastfilter.Filter; +import org.fastfilter.ribbon.RibbonFilter; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -53,7 +54,10 @@ public static List filters() { (StreamDeserializer) XorBinaryFuse16::deserialize}, new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, (Function) XorBinaryFuse32::deserialize, - (StreamDeserializer) XorBinaryFuse32::deserialize} + (StreamDeserializer) XorBinaryFuse32::deserialize}, + new Object[] {"RibbonFilter", (Function) RibbonFilter::construct, + (Function) RibbonFilter::deserialize, + (StreamDeserializer) RibbonFilter::deserialize} ); } From e13c35d3647a01fdcfb56cb28d699e346897e1cb Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Wed, 29 Jul 2026 23:22:11 +0530 Subject: [PATCH 36/38] Register RibbonFilter as FilterType.RIBBON / TestFilterType.RIBBON Setting is ignored the same way XOR_8/XOR_16/XOR_PLUS_8 ignore it here: TestAllFilters.test() hardcodes setting=10 for every TestFilterType, which would exceed RibbonFilter's 8-bit (byte-storage) ceiling if passed through. --- fastfilter/src/main/java/org/fastfilter/FilterType.java | 8 ++++++++ .../src/test/java/org/fastfilter/TestFilterType.java | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/fastfilter/src/main/java/org/fastfilter/FilterType.java b/fastfilter/src/main/java/org/fastfilter/FilterType.java index ac69e1a..ca837f7 100644 --- a/fastfilter/src/main/java/org/fastfilter/FilterType.java +++ b/fastfilter/src/main/java/org/fastfilter/FilterType.java @@ -87,6 +87,14 @@ public Filter construct(long[] keys, int setting) { return XorPlus8.construct(keys); } }, + RIBBON { + @Override + public Filter construct(long[] keys, int setting) { + // Fixed-width filter (like XOR_8/XOR_16/XOR_PLUS_8 above): setting + // is ignored, matching the r=8 default used for the Xor8 comparison. + return org.fastfilter.ribbon.RibbonFilter.construct(keys); + } + }, CUCKOO_8 { @Override public Filter construct(long[] keys, int setting) { diff --git a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java index bf3b172..125842b 100644 --- a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java +++ b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java @@ -11,6 +11,7 @@ import org.fastfilter.gcs.GolombCompressedSet; import org.fastfilter.gcs.GolombCompressedSet2; import org.fastfilter.mphf.MPHFilter; +import org.fastfilter.ribbon.RibbonFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -90,6 +91,12 @@ public Filter construct(long[] keys, int setting) { return XorBinaryFuse32.construct(keys); } }, + RIBBON { + @Override + public Filter construct(long[] keys, int setting) { + return RibbonFilter.construct(keys); + } + }, CUCKOO_8 { @Override public Filter construct(long[] keys, int setting) { From 48c063ecee8035f316a8fb0af9d4accfa5e3ce31 Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Thu, 30 Jul 2026 23:30:18 +0530 Subject: [PATCH 37/38] Rename RibbonFilter to RibbonNaiveFilter, RIBBON to RIBBON_NAIVE Reserves the RibbonFilter/RIBBON name for a future ICML-backed implementation with fast queries; this one keeps the naive O(w*r) per-column query, which is 70-100x slower than the Xor filters (see PR #54 benchmark). --- .../main/java/org/fastfilter/FilterType.java | 7 +++---- ...ibbonFilter.java => RibbonNaiveFilter.java} | 18 +++++++++--------- .../java/org/fastfilter/TestFilterType.java | 6 +++--- ...terTest.java => RibbonNaiveFilterTest.java} | 6 +++--- .../org/fastfilter/xor/SerializationTest.java | 8 ++++---- 5 files changed, 22 insertions(+), 23 deletions(-) rename fastfilter/src/main/java/org/fastfilter/ribbon/{RibbonFilter.java => RibbonNaiveFilter.java} (90%) rename fastfilter/src/test/java/org/fastfilter/ribbon/{RibbonFilterTest.java => RibbonNaiveFilterTest.java} (88%) diff --git a/fastfilter/src/main/java/org/fastfilter/FilterType.java b/fastfilter/src/main/java/org/fastfilter/FilterType.java index ca837f7..22ac83b 100644 --- a/fastfilter/src/main/java/org/fastfilter/FilterType.java +++ b/fastfilter/src/main/java/org/fastfilter/FilterType.java @@ -8,6 +8,7 @@ import org.fastfilter.cuckoo.CuckooPlus16; import org.fastfilter.cuckoo.CuckooPlus8; import org.fastfilter.gcs.GolombCompressedSet; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -87,12 +88,10 @@ public Filter construct(long[] keys, int setting) { return XorPlus8.construct(keys); } }, - RIBBON { + RIBBON_NAIVE { @Override public Filter construct(long[] keys, int setting) { - // Fixed-width filter (like XOR_8/XOR_16/XOR_PLUS_8 above): setting - // is ignored, matching the r=8 default used for the Xor8 comparison. - return org.fastfilter.ribbon.RibbonFilter.construct(keys); + return RibbonNaiveFilter.construct(keys); } }, CUCKOO_8 { diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java similarity index 90% rename from fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java rename to fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java index 5ec9a8f..deac226 100644 --- a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java @@ -20,7 +20,7 @@ * if all seed attempts at a given slot count fail - with more slots. * */ -public class RibbonFilter implements Filter { +public class RibbonNaiveFilter implements Filter { private static final int DEFAULT_RESULT_BITS = 8; private static final int COEFF_BITS = 64; @@ -36,7 +36,7 @@ public class RibbonFilter implements Filter { private final byte[] solution; private final int bitCount; - private RibbonFilter(int size, int resultBits, int numSlots, int numStarts, long seed, byte[] solution) { + private RibbonNaiveFilter(int size, int resultBits, int numSlots, int numStarts, long seed, byte[] solution) { this.size = size; this.resultBits = resultBits; this.numSlots = numSlots; @@ -46,11 +46,11 @@ private RibbonFilter(int size, int resultBits, int numSlots, int numStarts, long this.bitCount = numSlots * resultBits; } - public static RibbonFilter construct(long[] keys) { + public static RibbonNaiveFilter construct(long[] keys) { return construct(keys, DEFAULT_RESULT_BITS); } - public static RibbonFilter construct(long[] keys, int resultBits) { + public static RibbonNaiveFilter construct(long[] keys, int resultBits) { if (resultBits < 1 || resultBits > 8) { throw new IllegalArgumentException("resultBits must be between 1 and 8"); } @@ -68,7 +68,7 @@ public static RibbonFilter construct(long[] keys, int resultBits) { byte[] result = new byte[numSlots]; if (band(keys, seed, numStarts, resultBits, coeff, result)) { byte[] solution = backSubstitute(coeff, result, numSlots, resultBits); - return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); } } overheadFactor *= OVERHEAD_GROWTH_FACTOR; @@ -168,7 +168,7 @@ public void serialize(ByteBuffer buffer) { buffer.put(solution); } - public static RibbonFilter deserialize(ByteBuffer buffer) { + public static RibbonNaiveFilter deserialize(ByteBuffer buffer) { if (buffer.remaining() < Integer.BYTES * 5 + Long.BYTES) { throw new IllegalArgumentException("Buffer too small"); } @@ -183,7 +183,7 @@ public static RibbonFilter deserialize(ByteBuffer buffer) { } final byte[] solution = new byte[len]; buffer.get(solution); - return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); } @Override @@ -198,7 +198,7 @@ public void serialize(OutputStream out) throws IOException { dout.write(solution); } - public static RibbonFilter deserialize(InputStream in) throws IOException { + public static RibbonNaiveFilter deserialize(InputStream in) throws IOException { DataInputStream din = new DataInputStream(in); final int size = din.readInt(); final int resultBits = din.readInt(); @@ -208,6 +208,6 @@ public static RibbonFilter deserialize(InputStream in) throws IOException { final int len = din.readInt(); final byte[] solution = new byte[len]; din.readFully(solution); - return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); } } diff --git a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java index 125842b..5a2fb6b 100644 --- a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java +++ b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java @@ -11,7 +11,7 @@ import org.fastfilter.gcs.GolombCompressedSet; import org.fastfilter.gcs.GolombCompressedSet2; import org.fastfilter.mphf.MPHFilter; -import org.fastfilter.ribbon.RibbonFilter; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -91,10 +91,10 @@ public Filter construct(long[] keys, int setting) { return XorBinaryFuse32.construct(keys); } }, - RIBBON { + RIBBON_NAIVE { @Override public Filter construct(long[] keys, int setting) { - return RibbonFilter.construct(keys); + return RibbonNaiveFilter.construct(keys); } }, CUCKOO_8 { diff --git a/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java similarity index 88% rename from fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java rename to fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java index a150aba..6b1b9ad 100644 --- a/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java +++ b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java @@ -5,7 +5,7 @@ import org.fastfilter.utils.RandomGenerator; import org.junit.Test; -public class RibbonFilterTest { +public class RibbonNaiveFilterTest { @Test public void noFalseNegativesSmall() { @@ -29,7 +29,7 @@ public void falsePositiveRateNearExpected() { keys[i] = list[i]; nonKeys[i] = list[i + len]; } - RibbonFilter filter = RibbonFilter.construct(keys); + RibbonNaiveFilter filter = RibbonNaiveFilter.construct(keys); int falsePositives = 0; for (long nonKey : nonKeys) { if (filter.mayContain(nonKey)) { @@ -44,7 +44,7 @@ public void falsePositiveRateNearExpected() { private void assertNoFalseNegatives(int len) { long[] keys = new long[len]; RandomGenerator.createRandomUniqueListFast(keys, 0); - RibbonFilter filter = RibbonFilter.construct(keys); + RibbonNaiveFilter filter = RibbonNaiveFilter.construct(keys); for (long key : keys) { assertTrue("key " + key + " should be present", filter.mayContain(key)); } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java index b2435ec..2ce1d4e 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -13,7 +13,7 @@ import java.util.List; import java.util.function.Function; import org.fastfilter.Filter; -import org.fastfilter.ribbon.RibbonFilter; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -55,9 +55,9 @@ public static List filters() { new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, (Function) XorBinaryFuse32::deserialize, (StreamDeserializer) XorBinaryFuse32::deserialize}, - new Object[] {"RibbonFilter", (Function) RibbonFilter::construct, - (Function) RibbonFilter::deserialize, - (StreamDeserializer) RibbonFilter::deserialize} + new Object[] {"RibbonNaiveFilter", (Function) RibbonNaiveFilter::construct, + (Function) RibbonNaiveFilter::deserialize, + (StreamDeserializer) RibbonNaiveFilter::deserialize} ); } From 26baaf9d799acfe358d38a8c45c9d60798cbc19d Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Thu, 30 Jul 2026 23:30:36 +0530 Subject: [PATCH 38/38] Add class-level Javadoc citing the Ribbon filter papers --- .../org/fastfilter/ribbon/RibbonNaiveFilter.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java index deac226..c290a81 100644 --- a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java @@ -19,6 +19,20 @@ * dependence among the equations) and is retried with a new seed, and - * if all seed attempts at a given slot count fail - with more slots. * + * Martin Dietzfelbinger, Stefan Walzer, [Efficient Gauss Elimination for + * Near-Quadratic Matrices with One Short Random Block per + * Row](https://arxiv.org/abs/1907.04750), ESA 2019. + * + * Peter C. Dillinger, Stefan Walzer, [Ribbon filter: practically smaller + * than Bloom and Xor](https://arxiv.org/abs/2103.02515), arXiv:2103.02515, + * 2021. + * + * This is the "naive" query implementation: each lookup does an O(w * r) + * per-column dot product against the solved band, rather than the + * Interleaved Column-Major Layout (ICML) the papers above use for fast + * queries. It is kept under this name so a future ICML-backed + * implementation can use the plain RibbonFilter / RIBBON name once it + * exists. */ public class RibbonNaiveFilter implements Filter {