From 42aef913ba9a3d56a08cf83f8f5742b27ed52906 Mon Sep 17 00:00:00 2001 From: Vladimir Prus Date: Wed, 21 Jan 2026 10:32:45 +0000 Subject: [PATCH 1/9] Xor: implement stream serialization There are already methods operating on ByteBuffer, however versions taking streams allow to avoid data copying, and more directly compatible with various network APIs. Also, for reference, Guava Bloom filter implementation uses streams. There's a bit of code duplication as the result. Sadly, there's no standard class to wrap ByteBuffer in a stream, and pulling in an external dependency seems like an overkill here. --- .../src/main/java/org/fastfilter/Filter.java | 13 ++++ .../main/java/org/fastfilter/xor/Xor16.java | 28 ++++++++ .../main/java/org/fastfilter/xor/Xor8.java | 18 +++++ .../org/fastfilter/xor/XorBinaryFuse16.java | 30 ++++++++ .../org/fastfilter/xor/XorBinaryFuse32.java | 30 ++++++++ .../org/fastfilter/xor/XorBinaryFuse8.java | 26 +++++++ .../org/fastfilter/xor/SerializationTest.java | 70 +++++++++++++++++-- 7 files changed, 209 insertions(+), 6 deletions(-) diff --git a/fastfilter/src/main/java/org/fastfilter/Filter.java b/fastfilter/src/main/java/org/fastfilter/Filter.java index 05bfe43..83bd4f6 100644 --- a/fastfilter/src/main/java/org/fastfilter/Filter.java +++ b/fastfilter/src/main/java/org/fastfilter/Filter.java @@ -1,5 +1,7 @@ package org.fastfilter; +import java.io.IOException; +import java.io.OutputStream; import java.nio.ByteBuffer; /** @@ -85,4 +87,15 @@ default int getSerializedSize() { default void serialize(ByteBuffer buffer) { throw new UnsupportedOperationException(); } + + /** + * Serializes the filter state into the provided {@code OutputStream}. + * + * @param out the output stream where the serialized state of the filter will be written + * @throws IOException if writing to the stream fails + * @throws UnsupportedOperationException if the operation is not supported by the filter implementation + */ + default void serialize(OutputStream out) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java index 603a3f1..265ec12 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import org.fastfilter.Filter; @@ -199,4 +204,27 @@ public static Xor16 deserialize(ByteBuffer buffer) { return new Xor16(blockLength, bitCount, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(blockLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final short fp : fingerprints) { + dout.writeShort(fp); + } + } + + public static Xor16 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int blockLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readShort(); + } + final int bitCount = len * BITS_PER_FINGERPRINT; + return new Xor16(blockLength, bitCount, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java index 142fb87..4b7c5b6 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java @@ -241,4 +241,22 @@ public static Xor8 deserialize(ByteBuffer buffer) { return new Xor8(size, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(size); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + dout.write(fingerprints); + } + + public static Xor8 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int size = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] fingerprints = new byte[len]; + din.readFully(fingerprints); + return new Xor8(size, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java index b0ef76a..588e70c 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; import org.fastfilter.Filter; @@ -325,4 +330,29 @@ public static XorBinaryFuse16 deserialize(ByteBuffer buffer) { return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final short fp : fingerprints) { + dout.writeShort(fp); + } + } + + public static XorBinaryFuse16 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readShort(); + } + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java index fc1a01b..d226081 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -313,4 +318,29 @@ public static XorBinaryFuse32 deserialize(ByteBuffer buffer) { return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final int fp : fingerprints) { + dout.writeInt(fp); + } + } + + public static XorBinaryFuse32 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final int[] fingerprints = new int[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readInt(); + } + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java index e8f0337..f4e31ff 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java @@ -1,5 +1,10 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -321,4 +326,25 @@ public static XorBinaryFuse8 deserialize(ByteBuffer buffer) { return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints); } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + dout.write(fingerprints); + } + + public static XorBinaryFuse8 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] fingerprints = new byte[len]; + din.readFully(fingerprints); + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java index df6a204..023475d 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -5,6 +5,10 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; import java.util.function.Function; @@ -20,28 +24,36 @@ public class SerializationTest { private final String filterName; private final Function constructor; private final Function deserializer; + private final StreamDeserializer streamDeserializer; public SerializationTest(String filterName, Function constructor, - Function deserializer) { + Function deserializer, + StreamDeserializer streamDeserializer) { this.filterName = filterName; this.constructor = constructor; this.deserializer = deserializer; + this.streamDeserializer = streamDeserializer; } @Parameters(name = "{0}") public static List filters() { return List.of( new Object[] {"Xor8", (Function) Xor8::construct, - (Function) Xor8::deserialize}, + (Function) Xor8::deserialize, + (StreamDeserializer) Xor8::deserialize}, new Object[] {"Xor16", (Function) Xor16::construct, - (Function) Xor16::deserialize}, + (Function) Xor16::deserialize, + (StreamDeserializer) Xor16::deserialize}, new Object[] {"XorBinaryFuse8", (Function) XorBinaryFuse8::construct, - (Function) XorBinaryFuse8::deserialize}, + (Function) XorBinaryFuse8::deserialize, + (StreamDeserializer) XorBinaryFuse8::deserialize}, new Object[] {"XorBinaryFuse16", (Function) XorBinaryFuse16::construct, - (Function) XorBinaryFuse16::deserialize}, + (Function) XorBinaryFuse16::deserialize, + (StreamDeserializer) XorBinaryFuse16::deserialize}, new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, - (Function) XorBinaryFuse32::deserialize} + (Function) XorBinaryFuse32::deserialize, + (StreamDeserializer) XorBinaryFuse32::deserialize} ); } @@ -85,6 +97,52 @@ public void shouldSerializeAndDeserializeMediumFilter() { assertFalse("Key 1500L should not be in " + filterName + " filter", deserializedFilter.mayContain(1500L)); } + @Test + public void shouldSerializeAndDeserializeMediumFilterFromStream() throws IOException { + // Arrange + final var keys = new long[]{100L, 200L, 300L, 400L, 500L, 600L, 700L, 800L, 900L, 1000L}; + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + final var input = new ByteArrayInputStream(buffer.array()); + final var deserializedFilter = streamDeserializer.deserialize(input); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 50L should not be in " + filterName + " filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 1500L should not be in " + filterName + " filter", deserializedFilter.mayContain(1500L)); + } + + @Test + public void shouldSerializeToStreamAndDeserializeFromByteBuffer() throws IOException { + // Arrange + final var keys = new long[]{10L, 20L, 30L, 40L, 50L, 60L, 70L, 80L}; + final var originalFilter = constructor.apply(keys); + final var out = new ByteArrayOutputStream(); + + // Act + originalFilter.serialize(out); + final var buffer = ByteBuffer.wrap(out.toByteArray()); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 15L should not be in " + filterName + " filter", deserializedFilter.mayContain(15L)); + } + + @FunctionalInterface + private interface StreamDeserializer { + Filter deserialize(InputStream in) throws IOException; + } + @Test public void shouldSerializeAndDeserializeLargeFilter() { // Arrange From cbc2090773ca60a96a71e574e80493704570cbcb Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 23 Feb 2026 08:37:42 +0100 Subject: [PATCH 2/9] [maven-release-plugin] prepare release fastfilter_java-1.0.5 --- fastfilter/pom.xml | 2 +- jmh/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index 2e84df2..98257fa 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5-SNAPSHOT + 1.0.5 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index 4be23ea..fdb01a1 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5-SNAPSHOT + 1.0.5 4.0.0 diff --git a/pom.xml b/pom.xml index fbf3342..8703efb 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.github.fastfilter fastfilter_java pom - 1.0.5-SNAPSHOT + 1.0.5 fastfilter jmh @@ -47,7 +47,7 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - HEAD + fastfilter_java-1.0.5 From 21e1714720e34d1f0a69580fe701486959238427 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 23 Feb 2026 08:37:46 +0100 Subject: [PATCH 3/9] [maven-release-plugin] prepare for next development iteration --- fastfilter/pom.xml | 2 +- jmh/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index 98257fa..5178f9f 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5 + 1.0.6-SNAPSHOT 4.0.0 diff --git a/jmh/pom.xml b/jmh/pom.xml index fdb01a1..88db1ad 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -3,7 +3,7 @@ io.github.fastfilter fastfilter_java - 1.0.5 + 1.0.6-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 8703efb..727c8f2 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.github.fastfilter fastfilter_java pom - 1.0.5 + 1.0.6-SNAPSHOT fastfilter jmh @@ -47,7 +47,7 @@ scm:git:git@github.com:FastFilter/fastfilter_java.git scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - fastfilter_java-1.0.5 + HEAD From e1d04aed852e7c551bdae696bced7a1e75b5a4ba Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 26 Feb 2026 09:12:25 +0100 Subject: [PATCH 4/9] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3437b8a..895dea1 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ When using Maven: The latest version, 1.0.4, is not yet available on Maven centr io.github.fastfilter fastfilter - 1.0.4 + 1.0.5 ### Gradle @@ -110,7 +110,7 @@ When using Maven: The latest version, 1.0.4, is not yet available on Maven centr } dependencies { - implementation 'io.github.fastfilter.fastfilter_java:fastfilter:1.0.4' + implementation 'io.github.fastfilter.fastfilter_java:fastfilter:1.0.5' } ### Maven Central (version 1.0.2) From 993bc9851d1988ae9766f6303ac6df36bdd888f3 Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Wed, 29 Jul 2026 22:27:52 +0530 Subject: [PATCH 5/9] Add RibbonFilter core: banding, back-substitution, and query Implements the standard (non-homogeneous, w=64) Ribbon filter construction and membership query described in issue #33, following Xor8's structure (hash-derive, build-with-retries, query-by-recombine). --- .../org/fastfilter/ribbon/RibbonFilter.java | 144 ++++++++++++++++++ .../fastfilter/ribbon/RibbonFilterTest.java | 52 +++++++ 2 files changed, 196 insertions(+) create mode 100644 fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java create mode 100644 fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java new file mode 100644 index 0000000..138fe3d --- /dev/null +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java @@ -0,0 +1,144 @@ +package org.fastfilter.ribbon; + +import org.fastfilter.Filter; +import org.fastfilter.utils.Hash; + +/** + * A (non-homogeneous) Ribbon filter with a fixed ribbon width w = 64. + * + * A Ribbon filter is a retrieval data structure: for every key x in the + * set, solving a banded linear system over GF(2) yields a small function + * f(x) that returns an r-bit fingerprint. Construction can fail (linear + * dependence among the equations) and is retried with a new seed, and - + * if all seed attempts at a given slot count fail - with more slots. + * + */ +public class RibbonFilter implements Filter { + + private static final int DEFAULT_RESULT_BITS = 8; + private static final int COEFF_BITS = 64; + private static final int MAX_ATTEMPTS_PER_SIZE = 256; + private static final double INITIAL_OVERHEAD_FACTOR = 0.10; + private static final double OVERHEAD_GROWTH_FACTOR = 1.5; + + private final int size; + private final int resultBits; + private final int numSlots; + private final int numStarts; + private final long seed; + private final byte[] solution; + private final int bitCount; + + private RibbonFilter(int size, int resultBits, int numSlots, int numStarts, long seed, byte[] solution) { + this.size = size; + this.resultBits = resultBits; + this.numSlots = numSlots; + this.numStarts = numStarts; + this.seed = seed; + this.solution = solution; + this.bitCount = numSlots * resultBits; + } + + public static RibbonFilter construct(long[] keys) { + return construct(keys, DEFAULT_RESULT_BITS); + } + + public static RibbonFilter construct(long[] keys, int resultBits) { + if (resultBits < 1 || resultBits > 8) { + throw new IllegalArgumentException("resultBits must be between 1 and 8"); + } + int size = keys.length; + double overheadFactor = INITIAL_OVERHEAD_FACTOR; + while (true) { + int numStarts = (int) Math.ceil(size * (1 + overheadFactor)); + if (numStarts < 1) { + numStarts = 1; + } + int numSlots = numStarts + COEFF_BITS - 1; + for (int attempt = 0; attempt < MAX_ATTEMPTS_PER_SIZE; attempt++) { + long seed = Hash.randomSeed(); + long[] coeff = new long[numSlots]; + byte[] result = new byte[numSlots]; + if (band(keys, seed, numStarts, resultBits, coeff, result)) { + byte[] solution = backSubstitute(coeff, result, numSlots, resultBits); + return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + } + } + overheadFactor *= OVERHEAD_GROWTH_FACTOR; + } + } + + private static boolean band(long[] keys, long seed, int numStarts, int resultBits, long[] coeff, byte[] result) { + int mask = (1 << resultBits) - 1; + for (long key : keys) { + long hash = Hash.hash64(key, seed); + int s = Hash.reduce((int) hash, numStarts); + long c = Long.rotateLeft(hash, 21) | 1L; + int r = (int) (Long.rotateLeft(hash, 42) & mask); + while (true) { + int i = Long.numberOfTrailingZeros(c); + int p = s + i; + c >>>= i; + if (coeff[p] == 0) { + coeff[p] = c; + result[p] = (byte) r; + break; + } + c ^= coeff[p]; + r ^= result[p]; + s = p; + if (c == 0) { + return false; + } + } + } + return true; + } + + private static byte[] backSubstitute(long[] coeff, byte[] result, int numSlots, int resultBits) { + byte[] solution = new byte[numSlots]; + int mask = (1 << resultBits) - 1; + for (int i = numSlots - 1; i >= 0; i--) { + long c = coeff[i]; + if (c == 0) { + continue; + } + int contribution = 0; + long rest = c & ~1L; + while (rest != 0) { + int k = Long.numberOfTrailingZeros(rest); + rest &= rest - 1; + contribution ^= solution[i + k]; + } + solution[i] = (byte) ((result[i] ^ contribution) & mask); + } + return solution; + } + + @Override + public boolean mayContain(long key) { + long hash = Hash.hash64(key, seed); + int s = Hash.reduce((int) hash, numStarts); + long c = Long.rotateLeft(hash, 21) | 1L; + int mask = (1 << resultBits) - 1; + int expected = (int) (Long.rotateLeft(hash, 42) & mask); + + int actual = 0; + for (int j = 0; j < resultBits; j++) { + long columnBits = 0; + for (int k = 0; k < COEFF_BITS; k++) { + if (((solution[s + k] >>> j) & 1) != 0) { + columnBits |= (1L << k); + } + } + int bit = Long.bitCount(columnBits & c) & 1; + actual |= bit << j; + } + return actual == expected; + } + + @Override + public long getBitCount() { + return bitCount; + } +} diff --git a/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java new file mode 100644 index 0000000..a150aba --- /dev/null +++ b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java @@ -0,0 +1,52 @@ +package org.fastfilter.ribbon; + +import static org.junit.Assert.assertTrue; + +import org.fastfilter.utils.RandomGenerator; +import org.junit.Test; + +public class RibbonFilterTest { + + @Test + public void noFalseNegativesSmall() { + assertNoFalseNegatives(10); + assertNoFalseNegatives(1_000); + } + + @Test + public void noFalseNegativesLarge() { + assertNoFalseNegatives(1_000_000); + } + + @Test + public void falsePositiveRateNearExpected() { + int len = 1_000_000; + long[] list = new long[len * 2]; + RandomGenerator.createRandomUniqueListFast(list, 0); + long[] keys = new long[len]; + long[] nonKeys = new long[len]; + for (int i = 0; i < len; i++) { + keys[i] = list[i]; + nonKeys[i] = list[i + len]; + } + RibbonFilter filter = RibbonFilter.construct(keys); + int falsePositives = 0; + for (long nonKey : nonKeys) { + if (filter.mayContain(nonKey)) { + falsePositives++; + } + } + double fpp = (double) falsePositives / len; + // expected ~= 1/256 ~= 0.0039; generous margin to avoid flakiness + assertTrue("fpp too high: " + fpp, fpp < 0.01); + } + + private void assertNoFalseNegatives(int len) { + long[] keys = new long[len]; + RandomGenerator.createRandomUniqueListFast(keys, 0); + RibbonFilter filter = RibbonFilter.construct(keys); + for (long key : keys) { + assertTrue("key " + key + " should be present", filter.mayContain(key)); + } + } +} From f774f0f161a111e3fc960f35f77bb1beeba419dc Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Wed, 29 Jul 2026 22:33:00 +0530 Subject: [PATCH 6/9] Add RibbonFilter serialization and round-trip test coverage Mirrors Xor8's ByteBuffer/OutputStream serialize + static deserialize convention. Unlike Xor8, numSlots/numStarts must be stored explicitly (not re-derived from size) since the construction retry loop can grow the slot count beyond the simple sizing formula. --- .../org/fastfilter/ribbon/RibbonFilter.java | 69 +++++++++++++++++++ .../org/fastfilter/xor/SerializationTest.java | 6 +- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java index 138fe3d..5ec9a8f 100644 --- a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java @@ -1,5 +1,12 @@ package org.fastfilter.ribbon; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + import org.fastfilter.Filter; import org.fastfilter.utils.Hash; @@ -141,4 +148,66 @@ public boolean mayContain(long key) { public long getBitCount() { return bitCount; } + + @Override + public int getSerializedSize() { + return Integer.BYTES * 5 + Long.BYTES + solution.length * Byte.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + buffer.putInt(size); + buffer.putInt(resultBits); + buffer.putInt(numSlots); + buffer.putInt(numStarts); + buffer.putLong(seed); + buffer.putInt(solution.length); + buffer.put(solution); + } + + public static RibbonFilter deserialize(ByteBuffer buffer) { + if (buffer.remaining() < Integer.BYTES * 5 + Long.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + final int size = buffer.getInt(); + final int resultBits = buffer.getInt(); + final int numSlots = buffer.getInt(); + final int numStarts = buffer.getInt(); + final long seed = buffer.getLong(); + final int len = buffer.getInt(); + if (buffer.remaining() < len * Byte.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + final byte[] solution = new byte[len]; + buffer.get(solution); + return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + } + + @Override + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(size); + dout.writeInt(resultBits); + dout.writeInt(numSlots); + dout.writeInt(numStarts); + dout.writeLong(seed); + dout.writeInt(solution.length); + dout.write(solution); + } + + public static RibbonFilter deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int size = din.readInt(); + final int resultBits = din.readInt(); + final int numSlots = din.readInt(); + final int numStarts = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] solution = new byte[len]; + din.readFully(solution); + return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + } } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java index 023475d..b2435ec 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.function.Function; import org.fastfilter.Filter; +import org.fastfilter.ribbon.RibbonFilter; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -53,7 +54,10 @@ public static List filters() { (StreamDeserializer) XorBinaryFuse16::deserialize}, new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, (Function) XorBinaryFuse32::deserialize, - (StreamDeserializer) XorBinaryFuse32::deserialize} + (StreamDeserializer) XorBinaryFuse32::deserialize}, + new Object[] {"RibbonFilter", (Function) RibbonFilter::construct, + (Function) RibbonFilter::deserialize, + (StreamDeserializer) RibbonFilter::deserialize} ); } From e13c35d3647a01fdcfb56cb28d699e346897e1cb Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Wed, 29 Jul 2026 23:22:11 +0530 Subject: [PATCH 7/9] Register RibbonFilter as FilterType.RIBBON / TestFilterType.RIBBON Setting is ignored the same way XOR_8/XOR_16/XOR_PLUS_8 ignore it here: TestAllFilters.test() hardcodes setting=10 for every TestFilterType, which would exceed RibbonFilter's 8-bit (byte-storage) ceiling if passed through. --- fastfilter/src/main/java/org/fastfilter/FilterType.java | 8 ++++++++ .../src/test/java/org/fastfilter/TestFilterType.java | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/fastfilter/src/main/java/org/fastfilter/FilterType.java b/fastfilter/src/main/java/org/fastfilter/FilterType.java index ac69e1a..ca837f7 100644 --- a/fastfilter/src/main/java/org/fastfilter/FilterType.java +++ b/fastfilter/src/main/java/org/fastfilter/FilterType.java @@ -87,6 +87,14 @@ public Filter construct(long[] keys, int setting) { return XorPlus8.construct(keys); } }, + RIBBON { + @Override + public Filter construct(long[] keys, int setting) { + // Fixed-width filter (like XOR_8/XOR_16/XOR_PLUS_8 above): setting + // is ignored, matching the r=8 default used for the Xor8 comparison. + return org.fastfilter.ribbon.RibbonFilter.construct(keys); + } + }, CUCKOO_8 { @Override public Filter construct(long[] keys, int setting) { diff --git a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java index bf3b172..125842b 100644 --- a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java +++ b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java @@ -11,6 +11,7 @@ import org.fastfilter.gcs.GolombCompressedSet; import org.fastfilter.gcs.GolombCompressedSet2; import org.fastfilter.mphf.MPHFilter; +import org.fastfilter.ribbon.RibbonFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -90,6 +91,12 @@ public Filter construct(long[] keys, int setting) { return XorBinaryFuse32.construct(keys); } }, + RIBBON { + @Override + public Filter construct(long[] keys, int setting) { + return RibbonFilter.construct(keys); + } + }, CUCKOO_8 { @Override public Filter construct(long[] keys, int setting) { From 48c063ecee8035f316a8fb0af9d4accfa5e3ce31 Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Thu, 30 Jul 2026 23:30:18 +0530 Subject: [PATCH 8/9] Rename RibbonFilter to RibbonNaiveFilter, RIBBON to RIBBON_NAIVE Reserves the RibbonFilter/RIBBON name for a future ICML-backed implementation with fast queries; this one keeps the naive O(w*r) per-column query, which is 70-100x slower than the Xor filters (see PR #54 benchmark). --- .../main/java/org/fastfilter/FilterType.java | 7 +++---- ...ibbonFilter.java => RibbonNaiveFilter.java} | 18 +++++++++--------- .../java/org/fastfilter/TestFilterType.java | 6 +++--- ...terTest.java => RibbonNaiveFilterTest.java} | 6 +++--- .../org/fastfilter/xor/SerializationTest.java | 8 ++++---- 5 files changed, 22 insertions(+), 23 deletions(-) rename fastfilter/src/main/java/org/fastfilter/ribbon/{RibbonFilter.java => RibbonNaiveFilter.java} (90%) rename fastfilter/src/test/java/org/fastfilter/ribbon/{RibbonFilterTest.java => RibbonNaiveFilterTest.java} (88%) diff --git a/fastfilter/src/main/java/org/fastfilter/FilterType.java b/fastfilter/src/main/java/org/fastfilter/FilterType.java index ca837f7..22ac83b 100644 --- a/fastfilter/src/main/java/org/fastfilter/FilterType.java +++ b/fastfilter/src/main/java/org/fastfilter/FilterType.java @@ -8,6 +8,7 @@ import org.fastfilter.cuckoo.CuckooPlus16; import org.fastfilter.cuckoo.CuckooPlus8; import org.fastfilter.gcs.GolombCompressedSet; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -87,12 +88,10 @@ public Filter construct(long[] keys, int setting) { return XorPlus8.construct(keys); } }, - RIBBON { + RIBBON_NAIVE { @Override public Filter construct(long[] keys, int setting) { - // Fixed-width filter (like XOR_8/XOR_16/XOR_PLUS_8 above): setting - // is ignored, matching the r=8 default used for the Xor8 comparison. - return org.fastfilter.ribbon.RibbonFilter.construct(keys); + return RibbonNaiveFilter.construct(keys); } }, CUCKOO_8 { diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java similarity index 90% rename from fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java rename to fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java index 5ec9a8f..deac226 100644 --- a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonFilter.java +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java @@ -20,7 +20,7 @@ * if all seed attempts at a given slot count fail - with more slots. * */ -public class RibbonFilter implements Filter { +public class RibbonNaiveFilter implements Filter { private static final int DEFAULT_RESULT_BITS = 8; private static final int COEFF_BITS = 64; @@ -36,7 +36,7 @@ public class RibbonFilter implements Filter { private final byte[] solution; private final int bitCount; - private RibbonFilter(int size, int resultBits, int numSlots, int numStarts, long seed, byte[] solution) { + private RibbonNaiveFilter(int size, int resultBits, int numSlots, int numStarts, long seed, byte[] solution) { this.size = size; this.resultBits = resultBits; this.numSlots = numSlots; @@ -46,11 +46,11 @@ private RibbonFilter(int size, int resultBits, int numSlots, int numStarts, long this.bitCount = numSlots * resultBits; } - public static RibbonFilter construct(long[] keys) { + public static RibbonNaiveFilter construct(long[] keys) { return construct(keys, DEFAULT_RESULT_BITS); } - public static RibbonFilter construct(long[] keys, int resultBits) { + public static RibbonNaiveFilter construct(long[] keys, int resultBits) { if (resultBits < 1 || resultBits > 8) { throw new IllegalArgumentException("resultBits must be between 1 and 8"); } @@ -68,7 +68,7 @@ public static RibbonFilter construct(long[] keys, int resultBits) { byte[] result = new byte[numSlots]; if (band(keys, seed, numStarts, resultBits, coeff, result)) { byte[] solution = backSubstitute(coeff, result, numSlots, resultBits); - return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); } } overheadFactor *= OVERHEAD_GROWTH_FACTOR; @@ -168,7 +168,7 @@ public void serialize(ByteBuffer buffer) { buffer.put(solution); } - public static RibbonFilter deserialize(ByteBuffer buffer) { + public static RibbonNaiveFilter deserialize(ByteBuffer buffer) { if (buffer.remaining() < Integer.BYTES * 5 + Long.BYTES) { throw new IllegalArgumentException("Buffer too small"); } @@ -183,7 +183,7 @@ public static RibbonFilter deserialize(ByteBuffer buffer) { } final byte[] solution = new byte[len]; buffer.get(solution); - return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); } @Override @@ -198,7 +198,7 @@ public void serialize(OutputStream out) throws IOException { dout.write(solution); } - public static RibbonFilter deserialize(InputStream in) throws IOException { + public static RibbonNaiveFilter deserialize(InputStream in) throws IOException { DataInputStream din = new DataInputStream(in); final int size = din.readInt(); final int resultBits = din.readInt(); @@ -208,6 +208,6 @@ public static RibbonFilter deserialize(InputStream in) throws IOException { final int len = din.readInt(); final byte[] solution = new byte[len]; din.readFully(solution); - return new RibbonFilter(size, resultBits, numSlots, numStarts, seed, solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); } } diff --git a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java index 125842b..5a2fb6b 100644 --- a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java +++ b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java @@ -11,7 +11,7 @@ import org.fastfilter.gcs.GolombCompressedSet; import org.fastfilter.gcs.GolombCompressedSet2; import org.fastfilter.mphf.MPHFilter; -import org.fastfilter.ribbon.RibbonFilter; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -91,10 +91,10 @@ public Filter construct(long[] keys, int setting) { return XorBinaryFuse32.construct(keys); } }, - RIBBON { + RIBBON_NAIVE { @Override public Filter construct(long[] keys, int setting) { - return RibbonFilter.construct(keys); + return RibbonNaiveFilter.construct(keys); } }, CUCKOO_8 { diff --git a/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java similarity index 88% rename from fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java rename to fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java index a150aba..6b1b9ad 100644 --- a/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonFilterTest.java +++ b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java @@ -5,7 +5,7 @@ import org.fastfilter.utils.RandomGenerator; import org.junit.Test; -public class RibbonFilterTest { +public class RibbonNaiveFilterTest { @Test public void noFalseNegativesSmall() { @@ -29,7 +29,7 @@ public void falsePositiveRateNearExpected() { keys[i] = list[i]; nonKeys[i] = list[i + len]; } - RibbonFilter filter = RibbonFilter.construct(keys); + RibbonNaiveFilter filter = RibbonNaiveFilter.construct(keys); int falsePositives = 0; for (long nonKey : nonKeys) { if (filter.mayContain(nonKey)) { @@ -44,7 +44,7 @@ public void falsePositiveRateNearExpected() { private void assertNoFalseNegatives(int len) { long[] keys = new long[len]; RandomGenerator.createRandomUniqueListFast(keys, 0); - RibbonFilter filter = RibbonFilter.construct(keys); + RibbonNaiveFilter filter = RibbonNaiveFilter.construct(keys); for (long key : keys) { assertTrue("key " + key + " should be present", filter.mayContain(key)); } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java index b2435ec..2ce1d4e 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -13,7 +13,7 @@ import java.util.List; import java.util.function.Function; import org.fastfilter.Filter; -import org.fastfilter.ribbon.RibbonFilter; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -55,9 +55,9 @@ public static List filters() { new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, (Function) XorBinaryFuse32::deserialize, (StreamDeserializer) XorBinaryFuse32::deserialize}, - new Object[] {"RibbonFilter", (Function) RibbonFilter::construct, - (Function) RibbonFilter::deserialize, - (StreamDeserializer) RibbonFilter::deserialize} + new Object[] {"RibbonNaiveFilter", (Function) RibbonNaiveFilter::construct, + (Function) RibbonNaiveFilter::deserialize, + (StreamDeserializer) RibbonNaiveFilter::deserialize} ); } From 26baaf9d799acfe358d38a8c45c9d60798cbc19d Mon Sep 17 00:00:00 2001 From: amikumar91 Date: Thu, 30 Jul 2026 23:30:36 +0530 Subject: [PATCH 9/9] Add class-level Javadoc citing the Ribbon filter papers --- .../org/fastfilter/ribbon/RibbonNaiveFilter.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java index deac226..c290a81 100644 --- a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java @@ -19,6 +19,20 @@ * dependence among the equations) and is retried with a new seed, and - * if all seed attempts at a given slot count fail - with more slots. * + * Martin Dietzfelbinger, Stefan Walzer, [Efficient Gauss Elimination for + * Near-Quadratic Matrices with One Short Random Block per + * Row](https://arxiv.org/abs/1907.04750), ESA 2019. + * + * Peter C. Dillinger, Stefan Walzer, [Ribbon filter: practically smaller + * than Bloom and Xor](https://arxiv.org/abs/2103.02515), arXiv:2103.02515, + * 2021. + * + * This is the "naive" query implementation: each lookup does an O(w * r) + * per-column dot product against the solved band, rather than the + * Interleaved Column-Major Layout (ICML) the papers above use for fast + * queries. It is kept under this name so a future ICML-backed + * implementation can use the plain RibbonFilter / RIBBON name once it + * exists. */ public class RibbonNaiveFilter implements Filter {