diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 8d44d9d..0f1a00d 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -1,4 +1,4 @@ -name: Java 17 CI +name: Java 22 CI on: [push,pull_request] @@ -8,10 +8,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 17 - uses: actions/setup-java@v1 + - name: Set up JDK 22 + uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 22 + distribution: 'temurin' - name: Build run: mvn package - name: Test diff --git a/README.md b/README.md index efa19e2..895dea1 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The following filter types are currently implemented: * Xor filter: 8 and 16 bit variants; needs less space than cuckoo filters, with faster lookup * Xor+ filter: 8 and 16 bit variants; compressed xor filter -* Xor binary fuse filter: 8 and 32 bit variants; needs less space than xor filters, with faster lookup +* Xor binary fuse filter: 8. 16 and 32 bit variants; needs less space than xor filters (for large filters), with faster lookup * Cuckoo filter: 8 and 16 bit variants; uses cuckoo hashing to store fingerprints * Cuckoo+ filter: 8 and 16 bit variants, need a bit less space than regular cuckoo filters * Bloom filter: the 'standard' algorithm @@ -22,19 +22,108 @@ The following additional types are implemented, but less tested: ## Reference -* Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics (to appear). DOI: 10.1145/3510449 +* Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 * Thomas Mueller Graf, Daniel Lemire, [Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters](https://arxiv.org/abs/1912.08258), Journal of Experimental Algorithmics 25 (1), 2020. DOI: 10.1145/3376122 ## Usage -When using Maven: + +To use the XOR and Binary Fuse filters, first prepare an array of keys, then construct the filter: + +```java +import org.fastfilter.xor.XorBinaryFuse8; +import org.fastfilter.xor.XorBinaryFuse16; + +// Example keys +long[] keys = {1, 2, 3, 4, 5}; + +// Construct binary fuse filters= +XorBinaryFuse8 xorBinaryFuse8 = XorBinaryFuse8.construct(keys); +XorBinaryFuse16 xorBinaryFuse16 = XorBinaryFuse16.construct(keys); + +// Check membership +boolean mightContain = xor8.mayContain(1L); // true +boolean mightContain2 = xor8.mayContain(6L); // false (with high probability) +``` + +All filters implement the `Filter` interface and support the `mayContain(long key)` method to check if a key might be in the set. Note that false positives are possible, but false negatives are not. + +### Generating the Hash Values + +The library is written to process `long` values that are meant to be hash values. Though you do not need to use +cryptographically strong hashing, you should make sure that your hash functions are reasonable: they should +not generate too many collisions (two objects mapping to the same `long` value). + +### Serialization and Deserialization + +Filters can be serialized to and deserialized from a `ByteBuffer` for persistence or transmission: + +```java +import java.nio.ByteBuffer; + +// Assuming you have a constructed filter + +// Get the serialized size +int size = XorBinaryFuse8.getSerializedSize(); + +// Allocate a ByteBuffer +ByteBuffer buffer = ByteBuffer.allocate(size); + +// Serialize the filter +XorBinaryFuse8.serialize(buffer); + +// Prepare buffer for reading (flip) +buffer.flip(); + +// Deserialize the filter +XorBinaryFuse8 deserializedXorBinaryFuse8 = Xor8.deserialize(buffer); + +// The deserialized filter behaves identically to the original +``` + +This allows saving filters to files, databases, or sending them over networks. + +### Maven + +When using Maven: The latest version, 1.0.4, is not yet available on Maven central, see [issue #48](https://github.com/FastFilter/fastfilter_java/issues/48). However, it is available at https://jitpack.io/: + + + + jitpack.io + https://jitpack.io + + + + + io.github.fastfilter + fastfilter + 1.0.5 + + +### Gradle + + repositories { + mavenCentral() + maven { + url 'https://jitpack.io' + } + } + + dependencies { + implementation 'io.github.fastfilter.fastfilter_java:fastfilter:1.0.5' + } + +### Maven Central (version 1.0.2) + +The older version, 1.0.2, is available on Maven central. - io.github.fastfilter - fastfilter - 1.0.3 + io.github.fastfilter + fastfilter + 1.0.2 + # Other Xor Filter Implementations * [C](https://github.com/FastFilter/xor_singleheader) @@ -43,9 +132,12 @@ When using Maven: * [Erlang](https://github.com/mpope9/exor_filter) * [Go](https://github.com/FastFilter/xorfilter) * [Java](https://github.com/komiya-atsushi/xor-filter) -* [Python](https://github.com/GreyDireWolf/pyxorfilter) +* [Python](https://github.com/FastFilter/pyfusefilter) * Rust: [1](https://github.com/bnclabs/xorfilter), [2](https://github.com/codri/xorfilter-rs), [3](https://github.com/Polochon-street/rustxorfilter) * [C#](https://github.com/jonmat/FastIndex) +* [Java C wrapper](https://github.com/FastFilter/jfusebin) + +Note that the data format in other implementations may not match the data format in Java. ## Password Lookup Tool @@ -84,5 +176,72 @@ and with less than 1% probability "Found" or "Found; common". Internally, the tool uses a xor+ filter (see above) with 8 bits per fingerprint. Actually, 1024 smaller filters (segments) are made, the segment id being the highest 10 bits of the key. The lowest bit of the key is set to either 0 (regular) or 1 (common), and so two lookups are made per password. Because of that, the false positive rate is twice of what it would be with just one lookup (0.0078 instead of 0.0039). A regular Bloom filter with the same guarantees would be ~760 MB. For each lookup, one filter segment (so, less than 1 MB) are read from the file. +## Benchmarks + +The project includes JMH (Java Microbenchmark Harness) benchmarks to measure the performance of the filters. + +### Running Benchmarks + +#### Option 1: Run via Maven (recommended) + +To run the benchmarks directly from Maven (with minimal iterations for quick testing): + + mvn -pl jmh clean package exec:exec@run-benchmarks + +For full benchmarks, modify the pom.xml or run the JAR manually with custom parameters. + +This will compile and execute the JMH benchmarks for the XOR filters (XOR_8, XOR_16, XOR_BINARY_FUSE_8, XOR_BINARY_FUSE_16). + +#### Option 2: Run the JAR manually + +First, build the project: + + mvn clean package + +Then run the benchmarks: + + java -jar jmh/target/benchmarks.jar org.fastfilter.FilterBenchmark + +To run benchmarks for a specific filter type: + + java -jar jmh/target/benchmarks.jar org.fastfilter.FilterBenchmark -p filterType=XOR_BINARY_FUSE_8 + +Available filter types: `XOR_8`, `XOR_16`, `XOR_BINARY_FUSE_8`, `XOR_BINARY_FUSE_16`. + +### Benchmark Details + +The benchmarks measure: +- Average time per operation (nanoseconds) for lookups of existing and non-existing keys +- Throughput (operations per second) for the same operations +- False positive rate validation + + +Possible results: + +``` + +Benchmark (filterType) Mode Cnt Score Error Units +FilterBenchmark.benchmarkContainsExistingThroughput XOR_8 thrpt 412364492,755 ops/s +FilterBenchmark.benchmarkContainsExistingThroughput XOR_16 thrpt 397627818,837 ops/s +FilterBenchmark.benchmarkContainsExistingThroughput XOR_BINARY_FUSE_8 thrpt 516262004,459 ops/s +FilterBenchmark.benchmarkContainsExistingThroughput XOR_BINARY_FUSE_16 thrpt 489256453,340 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_8 thrpt 429856367,135 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_16 thrpt 441042890,257 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_BINARY_FUSE_8 thrpt 533609392,046 ops/s +FilterBenchmark.benchmarkContainsNonExistingThroughput XOR_BINARY_FUSE_16 thrpt 540058414,150 ops/s +FilterBenchmark.benchmarkContainsExisting XOR_8 avgt 2,475 ns/op +FilterBenchmark.benchmarkContainsExisting XOR_16 avgt 2,522 ns/op +FilterBenchmark.benchmarkContainsExisting XOR_BINARY_FUSE_8 avgt 1,965 ns/op +FilterBenchmark.benchmarkContainsExisting XOR_BINARY_FUSE_16 avgt 2,060 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_8 avgt 2,347 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_16 avgt 2,295 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_BINARY_FUSE_8 avgt 1,892 ns/op +FilterBenchmark.benchmarkContainsNonExisting XOR_BINARY_FUSE_16 avgt 1,903 ns/op +``` + +This indicates that we can issue about half a billion queries per second, and sustain a rate of about 2 ns per query. + +The benchmarks use 1,000,000 keys by default. You can modify the `NUM_KEYS` constant in `FilterBenchmark.java` for smaller/larger test sets. + diff --git a/fastfilter/pom.xml b/fastfilter/pom.xml index d1d96bb..5178f9f 100644 --- a/fastfilter/pom.xml +++ b/fastfilter/pom.xml @@ -1,11 +1,9 @@ - + io.github.fastfilter fastfilter_java - 1.0.3-SNAPSHOT + 1.0.6-SNAPSHOT 4.0.0 diff --git a/fastfilter/src/main/java/org/fastfilter/Filter.java b/fastfilter/src/main/java/org/fastfilter/Filter.java index 5eedbcb..83bd4f6 100644 --- a/fastfilter/src/main/java/org/fastfilter/Filter.java +++ b/fastfilter/src/main/java/org/fastfilter/Filter.java @@ -1,5 +1,9 @@ package org.fastfilter; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + /** * An approximate membership filter. */ @@ -14,7 +18,7 @@ public interface Filter { boolean mayContain(long key); /** - * Get the number of bits in thhe filter. + * Get the number of bits in the filter. * * @return the number of bits */ @@ -65,4 +69,33 @@ default long cardinality() { return -1; } + /** + * Get the serialized size of the filter. + * + * @return the size in bytes + */ + default int getSerializedSize() { + return -1; + } + + /** + * Serializes the filter state into the provided {@code ByteBuffer}. + * + * @param buffer the byte buffer where the serialized state of the filter will be written + * @throws UnsupportedOperationException if the operation is not supported by the filter implementation + */ + default void serialize(ByteBuffer buffer) { + throw new UnsupportedOperationException(); + } + + /** + * Serializes the filter state into the provided {@code OutputStream}. + * + * @param out the output stream where the serialized state of the filter will be written + * @throws IOException if writing to the stream fails + * @throws UnsupportedOperationException if the operation is not supported by the filter implementation + */ + default void serialize(OutputStream out) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/FilterType.java b/fastfilter/src/main/java/org/fastfilter/FilterType.java index efd2674..22ac83b 100644 --- a/fastfilter/src/main/java/org/fastfilter/FilterType.java +++ b/fastfilter/src/main/java/org/fastfilter/FilterType.java @@ -8,6 +8,7 @@ import org.fastfilter.cuckoo.CuckooPlus16; import org.fastfilter.cuckoo.CuckooPlus8; import org.fastfilter.gcs.GolombCompressedSet; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -57,18 +58,6 @@ public Filter construct(long[] keys, int setting) { return SuccinctCountingBlockedBloomRanked.construct(keys, setting); } }, - XOR_SIMPLE { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple.construct(keys); - } - }, - XOR_SIMPLE_2 { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple2.construct(keys); - } - }, XOR_8 { @Override public Filter construct(long[] keys, int setting) { @@ -99,6 +88,12 @@ public Filter construct(long[] keys, int setting) { return XorPlus8.construct(keys); } }, + RIBBON_NAIVE { + @Override + public Filter construct(long[] keys, int setting) { + return RibbonNaiveFilter.construct(keys); + } + }, CUCKOO_8 { @Override public Filter construct(long[] keys, int setting) { diff --git a/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java new file mode 100644 index 0000000..c290a81 --- /dev/null +++ b/fastfilter/src/main/java/org/fastfilter/ribbon/RibbonNaiveFilter.java @@ -0,0 +1,227 @@ +package org.fastfilter.ribbon; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +import org.fastfilter.Filter; +import org.fastfilter.utils.Hash; + +/** + * A (non-homogeneous) Ribbon filter with a fixed ribbon width w = 64. + * + * A Ribbon filter is a retrieval data structure: for every key x in the + * set, solving a banded linear system over GF(2) yields a small function + * f(x) that returns an r-bit fingerprint. Construction can fail (linear + * dependence among the equations) and is retried with a new seed, and - + * if all seed attempts at a given slot count fail - with more slots. + * + * Martin Dietzfelbinger, Stefan Walzer, [Efficient Gauss Elimination for + * Near-Quadratic Matrices with One Short Random Block per + * Row](https://arxiv.org/abs/1907.04750), ESA 2019. + * + * Peter C. Dillinger, Stefan Walzer, [Ribbon filter: practically smaller + * than Bloom and Xor](https://arxiv.org/abs/2103.02515), arXiv:2103.02515, + * 2021. + * + * This is the "naive" query implementation: each lookup does an O(w * r) + * per-column dot product against the solved band, rather than the + * Interleaved Column-Major Layout (ICML) the papers above use for fast + * queries. It is kept under this name so a future ICML-backed + * implementation can use the plain RibbonFilter / RIBBON name once it + * exists. + */ +public class RibbonNaiveFilter implements Filter { + + private static final int DEFAULT_RESULT_BITS = 8; + private static final int COEFF_BITS = 64; + private static final int MAX_ATTEMPTS_PER_SIZE = 256; + private static final double INITIAL_OVERHEAD_FACTOR = 0.10; + private static final double OVERHEAD_GROWTH_FACTOR = 1.5; + + private final int size; + private final int resultBits; + private final int numSlots; + private final int numStarts; + private final long seed; + private final byte[] solution; + private final int bitCount; + + private RibbonNaiveFilter(int size, int resultBits, int numSlots, int numStarts, long seed, byte[] solution) { + this.size = size; + this.resultBits = resultBits; + this.numSlots = numSlots; + this.numStarts = numStarts; + this.seed = seed; + this.solution = solution; + this.bitCount = numSlots * resultBits; + } + + public static RibbonNaiveFilter construct(long[] keys) { + return construct(keys, DEFAULT_RESULT_BITS); + } + + public static RibbonNaiveFilter construct(long[] keys, int resultBits) { + if (resultBits < 1 || resultBits > 8) { + throw new IllegalArgumentException("resultBits must be between 1 and 8"); + } + int size = keys.length; + double overheadFactor = INITIAL_OVERHEAD_FACTOR; + while (true) { + int numStarts = (int) Math.ceil(size * (1 + overheadFactor)); + if (numStarts < 1) { + numStarts = 1; + } + int numSlots = numStarts + COEFF_BITS - 1; + for (int attempt = 0; attempt < MAX_ATTEMPTS_PER_SIZE; attempt++) { + long seed = Hash.randomSeed(); + long[] coeff = new long[numSlots]; + byte[] result = new byte[numSlots]; + if (band(keys, seed, numStarts, resultBits, coeff, result)) { + byte[] solution = backSubstitute(coeff, result, numSlots, resultBits); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); + } + } + overheadFactor *= OVERHEAD_GROWTH_FACTOR; + } + } + + private static boolean band(long[] keys, long seed, int numStarts, int resultBits, long[] coeff, byte[] result) { + int mask = (1 << resultBits) - 1; + for (long key : keys) { + long hash = Hash.hash64(key, seed); + int s = Hash.reduce((int) hash, numStarts); + long c = Long.rotateLeft(hash, 21) | 1L; + int r = (int) (Long.rotateLeft(hash, 42) & mask); + while (true) { + int i = Long.numberOfTrailingZeros(c); + int p = s + i; + c >>>= i; + if (coeff[p] == 0) { + coeff[p] = c; + result[p] = (byte) r; + break; + } + c ^= coeff[p]; + r ^= result[p]; + s = p; + if (c == 0) { + return false; + } + } + } + return true; + } + + private static byte[] backSubstitute(long[] coeff, byte[] result, int numSlots, int resultBits) { + byte[] solution = new byte[numSlots]; + int mask = (1 << resultBits) - 1; + for (int i = numSlots - 1; i >= 0; i--) { + long c = coeff[i]; + if (c == 0) { + continue; + } + int contribution = 0; + long rest = c & ~1L; + while (rest != 0) { + int k = Long.numberOfTrailingZeros(rest); + rest &= rest - 1; + contribution ^= solution[i + k]; + } + solution[i] = (byte) ((result[i] ^ contribution) & mask); + } + return solution; + } + + @Override + public boolean mayContain(long key) { + long hash = Hash.hash64(key, seed); + int s = Hash.reduce((int) hash, numStarts); + long c = Long.rotateLeft(hash, 21) | 1L; + int mask = (1 << resultBits) - 1; + int expected = (int) (Long.rotateLeft(hash, 42) & mask); + + int actual = 0; + for (int j = 0; j < resultBits; j++) { + long columnBits = 0; + for (int k = 0; k < COEFF_BITS; k++) { + if (((solution[s + k] >>> j) & 1) != 0) { + columnBits |= (1L << k); + } + } + int bit = Long.bitCount(columnBits & c) & 1; + actual |= bit << j; + } + return actual == expected; + } + + @Override + public long getBitCount() { + return bitCount; + } + + @Override + public int getSerializedSize() { + return Integer.BYTES * 5 + Long.BYTES + solution.length * Byte.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + buffer.putInt(size); + buffer.putInt(resultBits); + buffer.putInt(numSlots); + buffer.putInt(numStarts); + buffer.putLong(seed); + buffer.putInt(solution.length); + buffer.put(solution); + } + + public static RibbonNaiveFilter deserialize(ByteBuffer buffer) { + if (buffer.remaining() < Integer.BYTES * 5 + Long.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + final int size = buffer.getInt(); + final int resultBits = buffer.getInt(); + final int numSlots = buffer.getInt(); + final int numStarts = buffer.getInt(); + final long seed = buffer.getLong(); + final int len = buffer.getInt(); + if (buffer.remaining() < len * Byte.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + final byte[] solution = new byte[len]; + buffer.get(solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); + } + + @Override + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(size); + dout.writeInt(resultBits); + dout.writeInt(numSlots); + dout.writeInt(numStarts); + dout.writeLong(seed); + dout.writeInt(solution.length); + dout.write(solution); + } + + public static RibbonNaiveFilter deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int size = din.readInt(); + final int resultBits = din.readInt(); + final int numSlots = din.readInt(); + final int numStarts = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] solution = new byte[len]; + din.readFully(solution); + return new RibbonNaiveFilter(size, resultBits, numSlots, numStarts, seed, solution); + } +} diff --git a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java index 6e6b02f..57fc3fe 100644 --- a/fastfilter/src/main/java/org/fastfilter/utils/Hash.java +++ b/fastfilter/src/main/java/org/fastfilter/utils/Hash.java @@ -3,7 +3,6 @@ import java.util.Random; public class Hash { - private static Random random = new Random(); public static void setSeed(long seed) { diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Deduplicator.java b/fastfilter/src/main/java/org/fastfilter/xor/Deduplicator.java new file mode 100644 index 0000000..6bd1759 --- /dev/null +++ b/fastfilter/src/main/java/org/fastfilter/xor/Deduplicator.java @@ -0,0 +1,27 @@ +package org.fastfilter.xor; + +import java.util.Arrays; + +public class Deduplicator { + + /** + * Sorts the keys array and removes duplicates in place. + * Returns the new length of the array (number of unique elements). + * + * @param keys the array of keys to deduplicate + * @param length the current length of the array + * @return the new length after removing duplicates + */ + public static int sortAndRemoveDup(long[] keys, int length) { + Arrays.sort(keys, 0, length); + int j = 1; + for (int i = 1; i < length; i++) { + if (keys[i] != keys[i - 1]) { + keys[j] = keys[i]; + j++; + } + } + return j; + } + +} diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java index bca40a6..265ec12 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor16.java @@ -1,10 +1,20 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + import org.fastfilter.Filter; import org.fastfilter.utils.Hash; /** + * The Xor16 filter implementation is experimental. We recommend using XorBinaryFuse16 instead. Use at your own risks. + * * The xor filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters](https://arxiv.org/abs/1912.08258), Journal of Experimental Algorithmics 25 (1), 2020. DOI: 10.1145/3376122 * * It needs 1.23 log(1/fpp) bits per key. It is related to the BDZ algorithm [1] * (a minimal perfect hash function algorithm). @@ -143,4 +153,78 @@ private int fingerprint(long hash) { return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1)); } + private Xor16(int blockLength, int bitCount, long seed, short[] fingerprints) { + this.blockLength = blockLength; + this.bitCount = bitCount; + this.seed = seed; + this.fingerprints = fingerprints; + } + + @Override + public int getSerializedSize() { + return Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Short.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(blockLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + for (final short fp : fingerprints) { + buffer.putShort(fp); + } + } + + public static Xor16 deserialize(ByteBuffer buffer) { + // Check minimum size for header (1 int + 1 long + 1 int for length) + if (buffer.remaining() < Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int blockLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Short.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = buffer.getShort(); + } + + final int bitCount = len * BITS_PER_FINGERPRINT; + + return new Xor16(blockLength, bitCount, seed, fingerprints); + } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(blockLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final short fp : fingerprints) { + dout.writeShort(fp); + } + } + + public static Xor16 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int blockLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readShort(); + } + final int bitCount = len * BITS_PER_FINGERPRINT; + return new Xor16(blockLength, bitCount, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java index 86ac870..4b7c5b6 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/Xor8.java @@ -1,12 +1,17 @@ package org.fastfilter.xor; import java.io.*; +import java.nio.ByteBuffer; import org.fastfilter.Filter; import org.fastfilter.utils.Hash; + /** + * The Xor8 filter implementation is experimental. We recommend using XorBinaryFuse8 instead. Use at your own risks. + * * The xor filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters](https://arxiv.org/abs/1912.08258), Journal of Experimental Algorithmics 25 (1), 2020. DOI: 10.1145/3376122 * * It needs 1.23 log(1/fpp) bits per key. It is related to the BDZ algorithm [1] * (a minimal perfect hash function algorithm). @@ -159,6 +164,7 @@ private int fingerprint(long hash) { return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1)); } + @Deprecated public byte[] getData() { try { ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -172,6 +178,7 @@ public byte[] getData() { } } + @Deprecated public Xor8(InputStream in) { try { DataInputStream din = new DataInputStream(in); @@ -187,4 +194,69 @@ public Xor8(InputStream in) { } } + private Xor8(int size, long seed, byte[] fingerprints) { + this.size = size; + this.arrayLength = getArrayLength(size); + this.bitCount = arrayLength * BITS_PER_FINGERPRINT; + this.blockLength = arrayLength / HASHES; + this.seed = seed; + this.fingerprints = fingerprints; + } + + @Override + public int getSerializedSize() { + return Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Byte.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(size); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + buffer.put(fingerprints); + } + + public static Xor8 deserialize(ByteBuffer buffer) { + // Check minimum size for header (1 int + 1 long + 1 int for length) + if (buffer.remaining() < Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int size = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Byte.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final byte[] fingerprints = new byte[len]; + buffer.get(fingerprints); + + return new Xor8(size, seed, fingerprints); + } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(size); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + dout.write(fingerprints); + } + + public static Xor8 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int size = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] fingerprints = new byte[len]; + din.readFully(fingerprints); + return new Xor8(size, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java new file mode 100644 index 0000000..588e70c --- /dev/null +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java @@ -0,0 +1,358 @@ +package org.fastfilter.xor; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.fastfilter.Filter; +import org.fastfilter.utils.Hash; + +/** + * The xor binary fuse filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 + */ +public class XorBinaryFuse16 implements Filter { + + private static final int ARITY = 3; + + private final int segmentCount; + private final int segmentCountLength; + private final int segmentLength; + private final int segmentLengthMask; + private final int arrayLength; + private final short[] fingerprints; + private long seed; + + private XorBinaryFuse16(int segmentCount, int segmentLength, long seed, short[] fingerprints) { + if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) { + throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength); + } + if (segmentCount <= 0) { + throw new IllegalArgumentException("Illegal segment count: " + segmentCount); + } + + this.segmentCount = segmentCount; + this.segmentCountLength = segmentCount * segmentLength; + this.segmentLength = segmentLength; + this.segmentLengthMask = segmentLength - 1; + this.arrayLength = fingerprints.length; + this.fingerprints = fingerprints; + this.seed = seed; + } + + public XorBinaryFuse16(int segmentCount, int segmentLength) { + this(segmentCount, segmentLength, 0L, new short[(segmentCount + ARITY - 1) * segmentLength]); + } + + public long getBitCount() { + return arrayLength * 16L; + } + + static int calculateSegmentLength(int arity, int size) { + int segmentLength; + if (arity == 3) { + segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(3.33) + 2.11); + } else if (arity == 4) { + segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(2.91) - 0.5); + } else { + // not supported + segmentLength = 65536; + } + return segmentLength; + } + + static double calculateSizeFactor(int arity, int size) { + double sizeFactor; + if (arity == 3) { + sizeFactor = Math.max(1.125, 0.875 + 0.25 * Math.log(1000000) / Math.log(size)); + } else if (arity == 4) { + sizeFactor = Math.max(1.075, 0.77 + 0.305 * Math.log(600000) / Math.log(size)); + } else { + // not supported + sizeFactor = 2.0; + } + return sizeFactor; + } + + private static int mod3(int x) { + if (x > 2) { + x -= 3; + } + return x; + } + + /** + * Constructs a new XorBinaryFuse16 filter from the given array of keys. + * The filter is designed to have a low false positive rate while being space-efficient. + * The keys array should contain unique values. The array may be mutated during construction + * (e.g., sorted and deduplicated) if the algorithm detects that there are likely too many duplicates. + * + * @param keys the array of long keys to add to the filter + * @return a new XorBinaryFuse16 filter containing all the keys + */ + public static XorBinaryFuse16 construct(long[] keys) { + int size = keys.length; + int segmentLength = calculateSegmentLength(ARITY, size); + // the current implementation hardcodes a 18-bit limit to + // to the segment length. + if (segmentLength > (1 << 18)) { + segmentLength = (1 << 18); + } + double sizeFactor = calculateSizeFactor(ARITY, size); + int capacity = (int) (size * sizeFactor); + int segmentCount = (capacity + segmentLength - 1) / segmentLength - (ARITY - 1); + int arrayLength = (segmentCount + ARITY - 1) * segmentLength; + segmentCount = (arrayLength + segmentLength - 1) / segmentLength; + segmentCount = segmentCount <= ARITY - 1 ? 1 : segmentCount - (ARITY - 1); + XorBinaryFuse16 filter = new XorBinaryFuse16(segmentCount, segmentLength); + filter.addAll(keys); + return filter; + } + + private void addAll(long[] keys) { + int size = keys.length; + long[] reverseOrder = new long[size + 1]; + byte[] reverseH = new byte[size]; + int reverseOrderPos = 0; + boolean duplicated = false; + + // the lowest 2 bits are the h index (0, 1, or 2) + // so we only have 6 bits for counting; + // but that's sufficient + byte[] t2count = new byte[arrayLength]; + long[] t2hash = new long[arrayLength]; + int[] alone = new int[arrayLength]; + int hashIndex = 0; + // the array h0, h1, h2, h0, h1, h2 + int[] h012 = new int[5]; + int blockBits = 1; + while ((1 << blockBits) < segmentCount) { + blockBits++; + } + int block = 1 << blockBits; + while (true) { + reverseOrder[size] = 1; + int[] startPos = new int[block]; + for (int i = 0; i < 1 << blockBits; i++) { + startPos[i] = (int) ((long) i * size / block); + } + // counting sort + + for(int i = 0; i < size; i++) { + long key = keys[i]; + long hash = Hash.hash64(key, seed); + int segmentIndex = (int) (hash >>> (64 - blockBits)); + // We only overwrite when the hash was zero. Zero hash values + // may be misplaced (unlikely). + while (reverseOrder[startPos[segmentIndex]] != 0) { + segmentIndex++; + segmentIndex &= (1 << blockBits) - 1; + } + reverseOrder[startPos[segmentIndex]] = hash; + startPos[segmentIndex]++; + } + byte countMask = 0; + for (int i = 0; i < size; i++) { + long hash = reverseOrder[i]; + for (int hi = 0; hi < 3; hi++) { + int index = getHashFromHash(hash, hi); + t2count[index] += 4; + t2count[index] ^= hi; + t2hash[index] ^= hash; + countMask |= t2count[index]; + } + } + startPos = null; + if (countMask >= 0) { + reverseOrderPos = 0; + int alonePos = 0; + for (int i = 0; i < arrayLength; i++) { + alone[alonePos] = i; + int inc = (t2count[i] >> 2) == 1 ? 1 : 0; + alonePos += inc; + } + + while (alonePos > 0) { + alonePos--; + int index = alone[alonePos]; + if ((t2count[index] >> 2) == 1) { + // It is still there! + long hash = t2hash[index]; + byte found = (byte) (t2count[index] & 3); + + reverseH[reverseOrderPos] = found; + reverseOrder[reverseOrderPos] = hash; + + h012[0] = getHashFromHash(hash, 0); + h012[1] = getHashFromHash(hash, 1); + h012[2] = getHashFromHash(hash, 2); + + int index3 = h012[mod3(found + 1)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 1); + t2hash[index3] ^= hash; + + index3 = h012[mod3(found + 2)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 2); + t2hash[index3] ^= hash; + + reverseOrderPos++; + } + } + } + if (reverseOrderPos == size) { + break; + } + hashIndex++; + Arrays.fill(t2count, (byte) 0); + Arrays.fill(t2hash, 0); + Arrays.fill(reverseOrder, 0); + // If we reach 10 passes, we assume that there are too many duplicates + // in the input key set. We then sort and remove duplicates in place. + // This should almost never happen. + if (countMask < 0 && !duplicated) { + size = Deduplicator.sortAndRemoveDup(keys, size); + duplicated = true; + } + if (hashIndex > 100) { + // if construction doesn't succeed eventually, + // then there is likely a problem with the hash function. + // It's better fail that either produce non-functional or incorrect filter. + throw new IllegalArgumentException("could not construct filter"); + } + // use a new random number + seed = Hash.randomSeed(); + } + + alone = null; + t2count = null; + t2hash = null; + + for (int i = reverseOrderPos - 1; i >= 0; i--) { + long hash = reverseOrder[i]; + int found = reverseH[i]; + short xor2 = fingerprint(hash); + h012[0] = getHashFromHash(hash, 0); + h012[1] = getHashFromHash(hash, 1); + h012[2] = getHashFromHash(hash, 2); + h012[3] = h012[0]; + h012[4] = h012[1]; + fingerprints[h012[found]] = (short) (xor2 ^ fingerprints[h012[found + 1]] ^ fingerprints[h012[found + 2]]); + } + } + + @Override + public boolean mayContain(long key) { + long hash = Hash.hash64(key, seed); + short f = fingerprint(hash); + int h0 = Hash.reduce((int) (hash >>> 32), segmentCountLength); + int h1 = h0 + segmentLength; + int h2 = h1 + segmentLength; + long hh = hash; + h1 ^= (int) ((hh >> 18) & segmentLengthMask); + h2 ^= (int) ((hh) & segmentLengthMask); + f ^= fingerprints[h0] ^ fingerprints[h1] ^ fingerprints[h2]; + return (f & 0xffff) == 0; + } + + @Override + public String toString() { + return "segmentLength " + segmentLength + " segmentCount " + segmentCount; + } + + int getHashFromHash(long hash, int index) { + long h = Hash.reduce((int) (hash >>> 32), segmentCountLength); + // long h = Hash.multiplyHighUnsigned(hash, segmentCountLength); + h += index * segmentLength; + // keep the lower 36 bits + long hh = hash & ((1L << 36) - 1); + // index 0: right shift by 36; index 1: right shift by 18; index 2: no shift + h ^= (int) ((hh >>> (36 - 18 * index)) & segmentLengthMask); + return (int) h; + } + + private short fingerprint(long hash) { + return (short) hash; + } + + @Override + public int getSerializedSize() { + return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Short.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(segmentLength); + buffer.putInt(segmentCountLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + for (final short fp : fingerprints) { + buffer.putShort(fp); + } + } + + public static XorBinaryFuse16 deserialize(ByteBuffer buffer) { + // Check minimum size for header (2 ints + 1 long + 1 int for length) + if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int segmentLength = buffer.getInt(); + final int segmentCountLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Short.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = buffer.getShort(); + } + + // Calculate segmentCount from segmentCountLength and segmentLength + final int segmentCount = segmentCountLength / segmentLength; + + return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints); + } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final short fp : fingerprints) { + dout.writeShort(fp); + } + } + + public static XorBinaryFuse16 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final short[] fingerprints = new short[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readShort(); + } + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints); + } +} diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java index d3f6125..d226081 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java @@ -1,12 +1,19 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Arrays; import org.fastfilter.Filter; import org.fastfilter.utils.Hash; /** - * The xor binary fuse filter, a new algorithm that can replace a Bloom filter. + * The XorBinaryFuse32 filter is experimental. We recommend using XorBinaryFuse8 or XorBinaryFuse16 instead. + * Use at your own risks. */ public class XorBinaryFuse32 implements Filter { @@ -20,19 +27,25 @@ public class XorBinaryFuse32 implements Filter { private final int[] fingerprints; private long seed; - public XorBinaryFuse32(int segmentCount, int segmentLength) { + private XorBinaryFuse32(int segmentCount, int segmentLength, long seed, int[] fingerprints) { if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) { throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength); } if (segmentCount <= 0) { throw new IllegalArgumentException("Illegal segment count: " + segmentCount); } - this.segmentLength = segmentLength; + this.segmentCount = segmentCount; - this.segmentLengthMask = segmentLength - 1; this.segmentCountLength = segmentCount * segmentLength; - this.arrayLength = (segmentCount + ARITY - 1) * segmentLength; - this.fingerprints = new int[arrayLength]; + this.segmentLength = segmentLength; + this.segmentLengthMask = segmentLength - 1; + this.arrayLength = fingerprints.length; + this.fingerprints = fingerprints; + this.seed = seed; + } + + public XorBinaryFuse32(int segmentCount, int segmentLength) { + this(segmentCount, segmentLength, 0L, new int[(segmentCount + ARITY - 1) * segmentLength]); } public long getBitCount() { @@ -200,12 +213,9 @@ private void addAll(long[] keys) { if (hashIndex > 100) { // if construction doesn't succeed eventually, - // then there is likely a problem with the hash function - // let us not crash the system: - for (int i = 0; i < fingerprints.length; i++) { - fingerprints[i] = (int) 0xFFFFFFFF; - } - return; + // then there is likely a problem with the hash function. + // It's better fail that either produce non-functional or incorrect filter. + throw new IllegalArgumentException("could not construct filter"); } // use a new random numbers seed = Hash.randomSeed(); @@ -261,4 +271,76 @@ private int fingerprint(long hash) { return (int) (hash ^ (hash >>> 32)); } + @Override + public int getSerializedSize() { + return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Integer.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(segmentLength); + buffer.putInt(segmentCountLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + for (final int fp : fingerprints) { + buffer.putInt(fp); + } + } + + public static XorBinaryFuse32 deserialize(ByteBuffer buffer) { + // Check minimum size for header (2 ints + 1 long + 1 int for length) + if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int segmentLength = buffer.getInt(); + final int segmentCountLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int[] fingerprints = new int[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = buffer.getInt(); + } + + // Calculate segmentCount from segmentCountLength and segmentLength + final int segmentCount = segmentCountLength / segmentLength; + + return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints); + } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + for (final int fp : fingerprints) { + dout.writeInt(fp); + } + } + + public static XorBinaryFuse32 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final int[] fingerprints = new int[len]; + for (int i = 0; i < len; i++) { + fingerprints[i] = din.readInt(); + } + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java index dfd5f45..f4e31ff 100644 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java +++ b/fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java @@ -1,5 +1,11 @@ package org.fastfilter.xor; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Arrays; import org.fastfilter.Filter; @@ -7,6 +13,7 @@ /** * The xor binary fuse filter, a new algorithm that can replace a Bloom filter. + * Thomas Mueller Graf, Daniel Lemire, [Binary Fuse Filters: Fast and Smaller Than Xor Filters](http://arxiv.org/abs/2201.01174), Journal of Experimental Algorithmics 27, 2022. DOI: 10.1145/3510449 */ public class XorBinaryFuse8 implements Filter { @@ -20,19 +27,25 @@ public class XorBinaryFuse8 implements Filter { private final byte[] fingerprints; private long seed; - public XorBinaryFuse8(int segmentCount, int segmentLength) { + private XorBinaryFuse8(int segmentCount, int segmentLength, long seed, byte[] fingerprints) { if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) { throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength); } if (segmentCount <= 0) { throw new IllegalArgumentException("Illegal segment count: " + segmentCount); } - this.segmentLength = segmentLength; + this.segmentCount = segmentCount; - this.segmentLengthMask = segmentLength - 1; this.segmentCountLength = segmentCount * segmentLength; - this.arrayLength = (segmentCount + ARITY - 1) * segmentLength; - this.fingerprints = new byte[arrayLength]; + this.segmentLength = segmentLength; + this.segmentLengthMask = segmentLength - 1; + this.arrayLength = fingerprints.length; + this.fingerprints = fingerprints; + this.seed = seed; + } + + public XorBinaryFuse8(int segmentCount, int segmentLength) { + this(segmentCount, segmentLength, 0L, new byte[(segmentCount + ARITY - 1) * segmentLength]); } public long getBitCount() { @@ -72,6 +85,15 @@ private static int mod3(int x) { return x; } + /** + * Constructs a new XorBinaryFuse8 filter from the given array of keys. + * The filter is designed to have a low false positive rate while being space-efficient. + * The keys array should contain unique values. The array may be mutated during construction + * (e.g., sorted and deduplicated) if the algorithm detects that there are likely too many duplicates. + * + * @param keys the array of long keys to add to the filter + * @return a new XorBinaryFuse8 filter containing all the keys + */ public static XorBinaryFuse8 construct(long[] keys) { int size = keys.length; int segmentLength = calculateSegmentLength(ARITY, size); @@ -96,6 +118,7 @@ private void addAll(long[] keys) { long[] reverseOrder = new long[size + 1]; byte[] reverseH = new byte[size]; int reverseOrderPos = 0; + boolean duplicated = false; // the lowest 2 bits are the h index (0, 1, or 2) // so we only have 6 bits for counting; @@ -111,7 +134,6 @@ private void addAll(long[] keys) { blockBits++; } int block = 1 << blockBits; - mainloop: while (true) { reverseOrder[size] = 1; int[] startPos = new int[block]; @@ -119,8 +141,8 @@ private void addAll(long[] keys) { startPos[i] = (int) ((long) i * size / block); } // counting sort - - for (long key : keys) { + for(int i = 0; i < size; i++) { + long key = keys[i]; long hash = Hash.hash64(key, seed); int segmentIndex = (int) (hash >>> (64 - blockBits)); // We only overwrite when the hash was zero. Zero hash values @@ -144,49 +166,46 @@ private void addAll(long[] keys) { } } startPos = null; - if (countMask < 0) { - // we have a possible counter overflow - continue mainloop; - } + if (countMask >= 0) { + reverseOrderPos = 0; + int alonePos = 0; + for (int i = 0; i < arrayLength; i++) { + alone[alonePos] = i; + int inc = (t2count[i] >> 2) == 1 ? 1 : 0; + alonePos += inc; + } - reverseOrderPos = 0; - int alonePos = 0; - for (int i = 0; i < arrayLength; i++) { - alone[alonePos] = i; - int inc = (t2count[i] >> 2) == 1 ? 1 : 0; - alonePos += inc; - } + while (alonePos > 0) { + alonePos--; + int index = alone[alonePos]; + if ((t2count[index] >> 2) == 1) { + // It is still there! + long hash = t2hash[index]; + byte found = (byte) (t2count[index] & 3); + + reverseH[reverseOrderPos] = found; + reverseOrder[reverseOrderPos] = hash; + + h012[0] = getHashFromHash(hash, 0); + h012[1] = getHashFromHash(hash, 1); + h012[2] = getHashFromHash(hash, 2); + + int index3 = h012[mod3(found + 1)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 1); + t2hash[index3] ^= hash; - while (alonePos > 0) { - alonePos--; - int index = alone[alonePos]; - if ((t2count[index] >> 2) == 1) { - // It is still there! - long hash = t2hash[index]; - byte found = (byte) (t2count[index] & 3); - - reverseH[reverseOrderPos] = found; - reverseOrder[reverseOrderPos] = hash; - - h012[0] = getHashFromHash(hash, 0); - h012[1] = getHashFromHash(hash, 1); - h012[2] = getHashFromHash(hash, 2); - - int index3 = h012[mod3(found + 1)]; - alone[alonePos] = index3; - alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); - t2count[index3] -= 4; - t2count[index3] ^= mod3(found + 1); - t2hash[index3] ^= hash; - - index3 = h012[mod3(found + 2)]; - alone[alonePos] = index3; - alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); - t2count[index3] -= 4; - t2count[index3] ^= mod3(found + 2); - t2hash[index3] ^= hash; - - reverseOrderPos++; + index3 = h012[mod3(found + 2)]; + alone[alonePos] = index3; + alonePos += ((t2count[index3] >> 2) == 2 ? 1 : 0); + t2count[index3] -= 4; + t2count[index3] ^= mod3(found + 2); + t2hash[index3] ^= hash; + + reverseOrderPos++; + } } } @@ -197,15 +216,18 @@ private void addAll(long[] keys) { Arrays.fill(t2count, (byte) 0); Arrays.fill(t2hash, 0); Arrays.fill(reverseOrder, 0); - + // If we reach 10 passes, we assume that there are too many duplicates + // in the input key set. We then sort and remove duplicates in place. + // This should almost never happen. + if (countMask < 0 && !duplicated) { + size = Deduplicator.sortAndRemoveDup(keys, size); + duplicated = true; + } if (hashIndex > 100) { // if construction doesn't succeed eventually, - // then there is likely a problem with the hash function - // let us not crash the system: - for(int i = 0; i < fingerprints.length; i++) { - fingerprints[i] = (byte)0xFF; - } - return; + // then there is likely a problem with the hash function. + // It's better fail that either produce non-functional or incorrect filter. + throw new IllegalArgumentException("could not construct filter"); } // use a new random numbers seed = Hash.randomSeed(); @@ -261,4 +283,68 @@ private byte fingerprint(long hash) { return (byte) hash; } + @Override + public int getSerializedSize() { + return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Byte.BYTES; + } + + @Override + public void serialize(ByteBuffer buffer) { + if (buffer.remaining() < getSerializedSize()) { + throw new IllegalArgumentException("Buffer too small"); + } + + buffer.putInt(segmentLength); + buffer.putInt(segmentCountLength); + buffer.putLong(seed); + buffer.putInt(fingerprints.length); + buffer.put(fingerprints); + } + + public static XorBinaryFuse8 deserialize(ByteBuffer buffer) { + // Check minimum size for header (2 ints + 1 long + 1 int for length) + if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final int segmentLength = buffer.getInt(); + final int segmentCountLength = buffer.getInt(); + final long seed = buffer.getLong(); + + final int len = buffer.getInt(); + + // Check if buffer has enough bytes for all fingerprints + if (buffer.remaining() < len * Byte.BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + + final byte[] fingerprints = new byte[len]; + buffer.get(fingerprints); + + // Calculate segmentCount from segmentCountLength and segmentLength + final int segmentCount = segmentCountLength / segmentLength; + + return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints); + } + + public void serialize(OutputStream out) throws IOException { + DataOutputStream dout = new DataOutputStream(out); + dout.writeInt(segmentLength); + dout.writeInt(segmentCountLength); + dout.writeLong(seed); + dout.writeInt(fingerprints.length); + dout.write(fingerprints); + } + + public static XorBinaryFuse8 deserialize(InputStream in) throws IOException { + DataInputStream din = new DataInputStream(in); + final int segmentLength = din.readInt(); + final int segmentCountLength = din.readInt(); + final long seed = din.readLong(); + final int len = din.readInt(); + final byte[] fingerprints = new byte[len]; + din.readFully(fingerprints); + final int segmentCount = segmentCountLength / segmentLength; + return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints); + } } diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple.java b/fastfilter/src/main/java/org/fastfilter/xor/XorSimple.java deleted file mode 100644 index 3165e94..0000000 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.fastfilter.xor; - -import java.util.Random; - -import org.fastfilter.Filter; -import org.fastfilter.utils.Hash; - -public class XorSimple implements Filter { - - private long seed; - private byte[] data; - int blockLength; - - public long getBitCount() { - return data.length * 8; - } - - public static XorSimple construct(long[] keys) { - return new XorSimple(keys); - } - - XorSimple(long[] keys) { - blockLength = (int) ((1.23 * keys.length) + 32) / 3; - data = new byte[3 * blockLength]; - while (true) { - seed = new Random().nextLong(); - long[] stack = new long[keys.length * 2]; - if (map(keys, seed, stack)) { - assign(stack, data); - return; - } - } - } - - boolean map(long[] keys, long seed, long[] stack) { - int[] C = new int[3 * blockLength]; - long[] H = new long[3 * blockLength]; - for (long k : keys) { - long x = Hash.hash64(k, seed); - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]++; - H[index] ^= x; - } - } - int[] Q = new int[3 * blockLength]; - int qi = 0; - for (int i = 0; i < C.length; i++) { - if (C[i] == 1) { - Q[qi++] = i; - } - } - int si = 0; - while (si < 2 * keys.length) { - int i = Q[--qi]; - if (C[i] == 1) { - long x = H[i]; - stack[si++] = x; - stack[si++] = i; - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]--; - if (C[index] == 1) { - Q[qi++] = index; - } - H[index] ^= x; - } - } - } - return si == 2 * keys.length; - } - - void assign(long[] stack, byte[] b) { - for(int stackPos = stack.length; stackPos > 0;) { - int index = (int) stack[--stackPos]; - long x = stack[--stackPos]; - b[index] = (byte) (fingerprint(x) ^ b[h(x, 0)] ^ b[h(x, 1)] ^ b[h(x, 2)]); - } - } - - int h(long x, int index) { - return Hash.reduce((int) Long.rotateLeft(x, index * 21), blockLength) + index * blockLength; - } - - @Override - public boolean mayContain(long key) { - long x = Hash.hash64(key, seed); - return fingerprint(x) == (data[h(x, 0)] ^ data[h(x, 1)] ^ data[h(x, 2)]); - } - - private byte fingerprint(long x) { - return (byte) x; - } - -} diff --git a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple2.java b/fastfilter/src/main/java/org/fastfilter/xor/XorSimple2.java deleted file mode 100644 index 8dd5339..0000000 --- a/fastfilter/src/main/java/org/fastfilter/xor/XorSimple2.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.fastfilter.xor; - -import org.fastfilter.utils.Hash; - -/** - * The same as xor simple, but with the alternative construction that ensures - * most empty entries are in the third part of the table. - */ -public class XorSimple2 extends XorSimple { - - public static XorSimple construct(long[] keys) { - return new XorSimple(keys); - } - - public XorSimple2(long[] keys) { - super(keys); - } - - boolean map(long[] keys, long seed, long[] stack) { - int[] C = new int[3 * blockLength]; - long[] H = new long[3 * blockLength]; - for (long k : keys) { - long x = Hash.hash64(k, seed); - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]++; - H[index] ^= x; - } - } - int[][] Q = new int[3][blockLength]; - int[] qi = new int[3]; - for (int i = 0; i < C.length; i++) { - if (C[i] == 1) { - int b = i / blockLength; - Q[b][qi[b]++] = i; - } - } - int si = 0; - while (si < 2 * keys.length && qi[0] > 0 || qi[1] > 0 || qi[2] > 0) { - int i; - if (qi[0] > 0) { - i = Q[0][--qi[0]]; - } else if (qi[1] > 0) { - i = Q[1][--qi[1]]; - } else if (qi[2] > 0) { - i = Q[2][--qi[2]]; - } else { - throw new AssertionError(); - } - if (C[i] == 1) { - long x = H[i]; - stack[si++] = x; - stack[si++] = i; - for (int j = 0; j < 3; j++) { - int index = h(x, j); - C[index]--; - if (C[index] == 1) { - int b = index / blockLength; - Q[b][qi[b]++] = index; - } - H[index] ^= x; - } - } - } - return si == 2 * keys.length; - } - -} diff --git a/fastfilter/src/test/java/org/fastfilter/RegressionTests.java b/fastfilter/src/test/java/org/fastfilter/RegressionTests.java index 289b5fd..12a160e 100644 --- a/fastfilter/src/test/java/org/fastfilter/RegressionTests.java +++ b/fastfilter/src/test/java/org/fastfilter/RegressionTests.java @@ -22,7 +22,6 @@ public static Object[][] regressionCases() { {SUCCINCT_COUNTING_BLOCKED_BLOOM, 6049486880293779298L, new long[]{1, 2, 3}, 8}, {SUCCINCT_COUNTING_BLOCKED_BLOOM, 353772444652436712L, new long[]{5828366214313827392L, -8467365400393984494L, -424469057572555653L}, 8}, // actually this one is impossible to reproduce because of the volatile seed - {XOR_SIMPLE, 6831634639270950343L, new long[]{1, 2, 3}, 8}, {CUCKOO_8, 6335419348330489927L, new long[]{1, 2, 3}, 8}, {CUCKOO_16, -9087718164446355442L, new long[]{1, 2, 3}, 8}, {CUCKOO_PLUS_8, -4031187722136552688L, new long[]{2173645522219008926L, 589862361776609381L, -1776331367981897399L, -7505626095864333717L, 6968992741301426055L, -3110009760358584538L, diff --git a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java index 749ab5f..5a2fb6b 100644 --- a/fastfilter/src/test/java/org/fastfilter/TestFilterType.java +++ b/fastfilter/src/test/java/org/fastfilter/TestFilterType.java @@ -11,6 +11,7 @@ import org.fastfilter.gcs.GolombCompressedSet; import org.fastfilter.gcs.GolombCompressedSet2; import org.fastfilter.mphf.MPHFilter; +import org.fastfilter.ribbon.RibbonNaiveFilter; import org.fastfilter.xor.*; import org.fastfilter.xorplus.XorPlus8; @@ -60,24 +61,6 @@ public Filter construct(long[] keys, int setting) { return SuccinctCountingBlockedBloomRanked.construct(keys, setting); } }, - XOR_SIMPLE { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple.construct(keys); - } - }, - XOR_SIMPLE_2 { - @Override - public Filter construct(long[] keys, int setting) { - return XorSimple2.construct(keys); - } - }, - XOR_8 { - @Override - public Filter construct(long[] keys, int setting) { - return Xor8.construct(keys); - } - }, XOR_16 { @Override public Filter construct(long[] keys, int setting) { @@ -96,12 +79,24 @@ public Filter construct(long[] keys, int setting) { return XorBinaryFuse8.construct(keys); } }, + XOR_BINARY_FUSE_16 { + @Override + public Filter construct(long[] keys, int setting) { + return XorBinaryFuse16.construct(keys); + } + }, XOR_BINARY_FUSE_32 { @Override public Filter construct(long[] keys, int setting) { return XorBinaryFuse32.construct(keys); } }, + RIBBON_NAIVE { + @Override + public Filter construct(long[] keys, int setting) { + return RibbonNaiveFilter.construct(keys); + } + }, CUCKOO_8 { @Override public Filter construct(long[] keys, int setting) { diff --git a/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java new file mode 100644 index 0000000..6b1b9ad --- /dev/null +++ b/fastfilter/src/test/java/org/fastfilter/ribbon/RibbonNaiveFilterTest.java @@ -0,0 +1,52 @@ +package org.fastfilter.ribbon; + +import static org.junit.Assert.assertTrue; + +import org.fastfilter.utils.RandomGenerator; +import org.junit.Test; + +public class RibbonNaiveFilterTest { + + @Test + public void noFalseNegativesSmall() { + assertNoFalseNegatives(10); + assertNoFalseNegatives(1_000); + } + + @Test + public void noFalseNegativesLarge() { + assertNoFalseNegatives(1_000_000); + } + + @Test + public void falsePositiveRateNearExpected() { + int len = 1_000_000; + long[] list = new long[len * 2]; + RandomGenerator.createRandomUniqueListFast(list, 0); + long[] keys = new long[len]; + long[] nonKeys = new long[len]; + for (int i = 0; i < len; i++) { + keys[i] = list[i]; + nonKeys[i] = list[i + len]; + } + RibbonNaiveFilter filter = RibbonNaiveFilter.construct(keys); + int falsePositives = 0; + for (long nonKey : nonKeys) { + if (filter.mayContain(nonKey)) { + falsePositives++; + } + } + double fpp = (double) falsePositives / len; + // expected ~= 1/256 ~= 0.0039; generous margin to avoid flakiness + assertTrue("fpp too high: " + fpp, fpp < 0.01); + } + + private void assertNoFalseNegatives(int len) { + long[] keys = new long[len]; + RandomGenerator.createRandomUniqueListFast(keys, 0); + RibbonNaiveFilter filter = RibbonNaiveFilter.construct(keys); + for (long key : keys) { + assertTrue("key " + key + " should be present", filter.mayContain(key)); + } + } +} diff --git a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java index 3039baf..20a9ab6 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse.java @@ -24,7 +24,6 @@ public class ProbabilityCFuse { public static void main(String... args) { for(int size = 1; size < 1_000_000; size *= 10) { - // for(int size = 1; size < 1_000_000; size = (size < 100) ? (size + 1) : (int) (size * 1.1)) { Data best = null; for (int segmentLengthBits = 3; segmentLengthBits < 14; segmentLengthBits++) { int segmentLength = 1 << segmentLengthBits; @@ -65,7 +64,6 @@ static Data getProbability(int size, int segmentLengthBits, double load, Data be if (best != null && d.bitsPerKey > best.bitsPerKey) { return null; } - // System.out.println(" test " + d); int successCount = 0; int testCount = Math.max(10, 10_000_000 / size); for(int seed = 0; seed < testCount; seed++) { diff --git a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java index 8a0bbe9..271414d 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/ProbabilityCFuse2.java @@ -37,12 +37,6 @@ private static void testProb() { long seed = 0; long key = i; long hash = Hash.hash64(key, seed); - -// int r0 = (int) Hash.hash64(hash, 1); -// int x = Hash.reduce(r0, segmentCount); -// int h0 = x + (int) (Hash.hash64(hash, 2) & (segmentLength - 1)); -// int h1 = x + (int) (Hash.hash64(hash, 3) & (segmentLength - 1)); -// int h2 = x + (int) (Hash.hash64(hash, 4) & (segmentLength - 1)); // int r0 = (int) Hash.hash64(hash, 1); int x = Hash.reduce(r0, segmentCount * 2 + segmentLength - 1); @@ -63,16 +57,9 @@ private static void testProb() { } public static void main(String... args) { -//testProb(); -//if(true)return; - - -// for(int size = 100_000; size < 1_000_000; size *= 10) { for(int size = 1; size < 1_000_000; size *= 10) { - // for(int size = 1; size < 1_000_000; size = (size < 100) ? (size + 1) : (int) (size * 1.1)) { Data best = null; for (int segmentLengthBits = 3; segmentLengthBits <= 12; segmentLengthBits++) { -// for (int segmentLengthBits = 3; segmentLengthBits < 14; segmentLengthBits++) { int segmentLength = 1 << segmentLengthBits; if (segmentLength > size) { break; @@ -89,9 +76,6 @@ public static void main(String... args) { } if (best != null) { System.out.println(best); -// for(int i=0; i<100; i++) { -// System.out.println(i + ": " + best.data[i]); -// } } } } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java new file mode 100644 index 0000000..2ce1d4e --- /dev/null +++ b/fastfilter/src/test/java/org/fastfilter/xor/SerializationTest.java @@ -0,0 +1,360 @@ +package org.fastfilter.xor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.function.Function; +import org.fastfilter.Filter; +import org.fastfilter.ribbon.RibbonNaiveFilter; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class SerializationTest { + + private final String filterName; + private final Function constructor; + private final Function deserializer; + private final StreamDeserializer streamDeserializer; + + public SerializationTest(String filterName, + Function constructor, + Function deserializer, + StreamDeserializer streamDeserializer) { + this.filterName = filterName; + this.constructor = constructor; + this.deserializer = deserializer; + this.streamDeserializer = streamDeserializer; + } + + @Parameters(name = "{0}") + public static List filters() { + return List.of( + new Object[] {"Xor8", (Function) Xor8::construct, + (Function) Xor8::deserialize, + (StreamDeserializer) Xor8::deserialize}, + new Object[] {"Xor16", (Function) Xor16::construct, + (Function) Xor16::deserialize, + (StreamDeserializer) Xor16::deserialize}, + new Object[] {"XorBinaryFuse8", (Function) XorBinaryFuse8::construct, + (Function) XorBinaryFuse8::deserialize, + (StreamDeserializer) XorBinaryFuse8::deserialize}, + new Object[] {"XorBinaryFuse16", (Function) XorBinaryFuse16::construct, + (Function) XorBinaryFuse16::deserialize, + (StreamDeserializer) XorBinaryFuse16::deserialize}, + new Object[] {"XorBinaryFuse32", (Function) XorBinaryFuse32::construct, + (Function) XorBinaryFuse32::deserialize, + (StreamDeserializer) XorBinaryFuse32::deserialize}, + new Object[] {"RibbonNaiveFilter", (Function) RibbonNaiveFilter::construct, + (Function) RibbonNaiveFilter::deserialize, + (StreamDeserializer) RibbonNaiveFilter::deserialize} + ); + } + + @Test + public void shouldSerializeAndDeserializeSmallFilter() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L}; + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + } + + @Test + public void shouldSerializeAndDeserializeMediumFilter() { + // Arrange + final var keys = new long[]{100L, 200L, 300L, 400L, 500L, 600L, 700L, 800L, 900L, 1000L}; + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 50L should not be in " + filterName + " filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 1500L should not be in " + filterName + " filter", deserializedFilter.mayContain(1500L)); + } + + @Test + public void shouldSerializeAndDeserializeMediumFilterFromStream() throws IOException { + // Arrange + final var keys = new long[]{100L, 200L, 300L, 400L, 500L, 600L, 700L, 800L, 900L, 1000L}; + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + final var input = new ByteArrayInputStream(buffer.array()); + final var deserializedFilter = streamDeserializer.deserialize(input); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 50L should not be in " + filterName + " filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 1500L should not be in " + filterName + " filter", deserializedFilter.mayContain(1500L)); + } + + @Test + public void shouldSerializeToStreamAndDeserializeFromByteBuffer() throws IOException { + // Arrange + final var keys = new long[]{10L, 20L, 30L, 40L, 50L, 60L, 70L, 80L}; + final var originalFilter = constructor.apply(keys); + final var out = new ByteArrayOutputStream(); + + // Act + originalFilter.serialize(out); + final var buffer = ByteBuffer.wrap(out.toByteArray()); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + assertFalse("Key 15L should not be in " + filterName + " filter", deserializedFilter.mayContain(15L)); + } + + @FunctionalInterface + private interface StreamDeserializer { + Filter deserialize(InputStream in) throws IOException; + } + + @Test + public void shouldSerializeAndDeserializeLargeFilter() { + // Arrange + final int size = 10000; + final var keys = new long[size]; + for (int i = 0; i < size; i++) { + keys[i] = i * 100L; + } + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (int i = 0; i < size; i++) { + final long key = i * 100L; + assertTrue("Key " + key + " should be present in deserialized " + filterName + " filter", + deserializedFilter.mayContain(key)); + } + // Test some keys that should not be in the filter + assertFalse("Key 1L should not be in filter", deserializedFilter.mayContain(1L)); + assertFalse("Key 50L should not be in filter", deserializedFilter.mayContain(50L)); + assertFalse("Key 99L should not be in filter", deserializedFilter.mayContain(99L)); + } + + @Test + public void shouldPreserveFilterCharacteristicsAfterDeserialization() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L}; + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + assertEquals("Bit count should be preserved for " + filterName, + originalFilter.getBitCount(), deserializedFilter.getBitCount()); + assertEquals("Serialized size should be preserved for " + filterName, + originalFilter.getSerializedSize(), deserializedFilter.getSerializedSize()); + } + + @Test + public void shouldHandleMultipleSerializationRounds() { + // Arrange + final var keys = new long[]{10L, 20L, 30L, 40L, 50L}; + final var originalFilter = constructor.apply(keys); + final var buffer1 = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act - First round + originalFilter.serialize(buffer1); + buffer1.flip(); + final var filter1 = deserializer.apply(buffer1); + + // Act - Second round + final var buffer2 = ByteBuffer.allocate(filter1.getSerializedSize()); + filter1.serialize(buffer2); + buffer2.flip(); + final var filter2 = deserializer.apply(buffer2); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present after first deserialization of " + filterName, + filter1.mayContain(key)); + assertTrue("Key " + key + " should be present after second deserialization of " + filterName, + filter2.mayContain(key)); + } + } + + @Test + public void shouldThrowExceptionWhenSerializeBufferTooSmall() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L}; + final var filter = constructor.apply(keys); + final var smallBuffer = ByteBuffer.allocate(filter.getSerializedSize() - 1); + + // Act & Assert + try { + filter.serialize(smallBuffer); + fail("Should have thrown IllegalArgumentException for buffer too small"); + } catch (IllegalArgumentException e) { + assertEquals("Buffer too small", e.getMessage()); + } + } + + @Test + public void shouldThrowExceptionWhenDeserializeBufferTooSmall() { + // Arrange + final var tooSmallBuffer = ByteBuffer.allocate(10); + + // Act & Assert + try { + deserializer.apply(tooSmallBuffer); + fail("Should have thrown IllegalArgumentException for buffer too small"); + } catch (IllegalArgumentException e) { + assertEquals("Buffer too small", e.getMessage()); + } + } + + @Test + public void shouldHandleFilterWithSequentialKeys() { + // Arrange + final int size = 1000; + final var keys = new long[size]; + for (int i = 0; i < size; i++) { + keys[i] = i; + } + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (int i = 0; i < size; i++) { + assertTrue("Sequential key " + i + " should be present in " + filterName, + deserializedFilter.mayContain(i)); + } + assertFalse("Key outside range should not be in " + filterName + " filter", + deserializedFilter.mayContain(size + 1000)); + } + + @Test + public void shouldHandleFilterWithRandomLargeKeys() { + // Arrange + final var keys = new long[]{ + Long.MAX_VALUE - 1, + Long.MAX_VALUE - 100, + Long.MAX_VALUE - 1000, + Long.MAX_VALUE / 2, + Long.MAX_VALUE / 3 + }; + final var originalFilter = constructor.apply(keys); + final var buffer = ByteBuffer.allocate(originalFilter.getSerializedSize()); + + // Act + originalFilter.serialize(buffer); + buffer.flip(); + final var deserializedFilter = deserializer.apply(buffer); + + // Assert + for (final long key : keys) { + assertTrue("Large key " + key + " should be present in " + filterName, + deserializedFilter.mayContain(key)); + } + } + + @Test + public void shouldCorrectlyCalculateSerializedSize() { + // Arrange + final var keys = new long[]{1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L}; + final var filter = constructor.apply(keys); + final int expectedSizeInBytes = filter.getSerializedSize(); + final var buffer = ByteBuffer.allocate(expectedSizeInBytes); + + // Act + filter.serialize(buffer); + + // Assert + assertEquals("Buffer position should equal serialized size for " + filterName, + expectedSizeInBytes, buffer.position()); + assertEquals("Buffer should have no remaining space for " + filterName, + 0, buffer.remaining()); + } + + @Test + public void shouldHandleExactBufferSize() { + // Arrange + final var keys = new long[]{100L, 200L, 300L}; + final var filter = constructor.apply(keys); + final var exactBuffer = ByteBuffer.allocate(filter.getSerializedSize()); + + // Act + filter.serialize(exactBuffer); + exactBuffer.flip(); + final var deserializedFilter = deserializer.apply(exactBuffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present with exact buffer in " + filterName, + deserializedFilter.mayContain(key)); + } + assertEquals("No bytes should remain in buffer for " + filterName, 0, exactBuffer.remaining()); + } + + @Test + public void shouldHandleLargerBufferThanNeeded() { + // Arrange + final var keys = new long[]{1L, 2L, 3L}; + final var filter = constructor.apply(keys); + final var largeBuffer = ByteBuffer.allocate(filter.getSerializedSize() + 1000); + + // Act + filter.serialize(largeBuffer); + largeBuffer.flip(); + final var deserializedFilter = deserializer.apply(largeBuffer); + + // Assert + for (final long key : keys) { + assertTrue("Key " + key + " should be present with larger buffer in " + filterName, + deserializedFilter.mayContain(key)); + } + } +} diff --git a/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java b/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java index 2c15a99..5fc0d93 100644 --- a/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java +++ b/fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java @@ -13,9 +13,8 @@ public void small() { Xor8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); Xor16.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); XorBinaryFuse8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); + XorBinaryFuse16.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); XorBinaryFuse32.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); - XorSimple.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); - XorSimple2.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL}); } @Test @@ -64,6 +63,19 @@ public void smallSizes() { } } + @Test + public void smallSizes16() { + long lastTime = System.currentTimeMillis(); + for (int n = 1; n < 1_500_000; n = (int) ((n * 1.01) + 7)) { + XorBinaryFuse16 f = testWithSize16(n); + long now = System.currentTimeMillis(); + if (now - lastTime > 5000) { + lastTime = now; + System.out.println("n=" + n + " " + f.toString()); + } + } + } + private static XorBinaryFuse8 testWithSize(int n) { long[] keys = new long[n]; for (int i = 0; i < n; i++) { @@ -72,4 +84,12 @@ private static XorBinaryFuse8 testWithSize(int n) { return XorBinaryFuse8.construct(keys); } + private static XorBinaryFuse16 testWithSize16(int n) { + long[] keys = new long[n]; + for (int i = 0; i < n; i++) { + keys[i] = i; + } + return XorBinaryFuse16.construct(keys); + } + } diff --git a/fastfilter/src/test/java/org/fastfilter/xor/StringFilters.java b/fastfilter/src/test/java/org/fastfilter/xor/StringFilters.java new file mode 100644 index 0000000..35f7324 --- /dev/null +++ b/fastfilter/src/test/java/org/fastfilter/xor/StringFilters.java @@ -0,0 +1,131 @@ +package org.fastfilter.xor; + +import static org.junit.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Random; +import java.util.Set; + +import org.fastfilter.Filter; +import org.junit.Test; + +public class StringFilters { + + private static final int NUM_STRINGS = 100_000; + private static final int NUM_TEST_STRINGS = 1_000; + private static final Random random = new Random(42); + + private static final long[] keys = generateKeys(); + private static final long[] testKeys = generateTestKeys(); + + private static long[] generateKeys() { + String[] strings = new String[NUM_STRINGS]; + for (int i = 0; i < NUM_STRINGS; i++) { + strings[i] = generateRandomString(); + } + long[] k = new long[NUM_STRINGS]; + for (int i = 0; i < NUM_STRINGS; i++) { + k[i] = hashString(strings[i]); + } + checkUniqueness(k, "keys"); + return k; + } + + private static long[] generateTestKeys() { + String[] strings = new String[NUM_TEST_STRINGS]; + for (int i = 0; i < NUM_TEST_STRINGS; i++) { + strings[i] = generateRandomString(); + } + long[] k = new long[NUM_TEST_STRINGS]; + for (int i = 0; i < NUM_TEST_STRINGS; i++) { + k[i] = hashString(strings[i]); + } + checkUniqueness(k, "test keys"); + return k; + } + + private static void checkUniqueness(long[] array, String name) { + Set set = new HashSet<>(); + int collisions = 0; + for (long l : array) { + if (!set.add(l)) { + collisions++; + } + } + if (collisions > 0) { + System.out.println("Warning: " + collisions + " hash collisions in " + name); + } else { + System.out.println("No hash collisions in " + name); + } + } + + private static String generateRandomString() { + int length = 5 + random.nextInt(16); // 5 to 20 chars + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + sb.append((char) ('a' + random.nextInt(26))); + } + return sb.toString(); + } + + private static long hashString(String s) { + long h = 0; + for (char c : s.toCharArray()) { + h = h * 31 + c; + } + return h; + } + + @Test + public void testXor8() { + testFilter(Xor8.class); + } + + @Test + public void testXor16() { + testFilter(Xor16.class); + } + + @Test + public void testXorBinaryFuse8() { + testFilter(XorBinaryFuse8.class); + } + + @Test + public void testXorBinaryFuse16() { + testFilter(XorBinaryFuse16.class); + } + + @Test + public void testXorBinaryFuse32() { + testFilter(XorBinaryFuse32.class); + } + + private void testFilter(Class filterClass) { + // Construct filter + Filter filter; + try { + filter = (Filter) filterClass.getMethod("construct", long[].class).invoke(null, (Object) keys); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // Check all keys are in the filter + for (int i = 0; i < NUM_STRINGS; i++) { + assertTrue("Key " + i + " should be in filter", filter.mayContain(keys[i])); + } + + // Check false positives on test keys + int falsePositives = 0; + for (int i = 0; i < NUM_TEST_STRINGS; i++) { + if (filter.mayContain(testKeys[i])) { + falsePositives++; + } + } + + // Expect low false positive rate (less than 1% for most filters) + double fpp = (double) falsePositives / NUM_TEST_STRINGS; + System.out.println(filterClass.getSimpleName() + " false positive rate: " + fpp); + assertTrue("False positive rate should be low: " + fpp, fpp < 0.01); // Allow up to 1% + } +} diff --git a/jmh/pom.xml b/jmh/pom.xml index 7c7392a..88db1ad 100644 --- a/jmh/pom.xml +++ b/jmh/pom.xml @@ -1,11 +1,9 @@ - + io.github.fastfilter fastfilter_java - 1.0.3-SNAPSHOT + 1.0.6-SNAPSHOT 4.0.0 @@ -41,6 +39,13 @@ true ${maven.compiler.source} ${maven.compiler.target} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + true true true @@ -69,6 +74,34 @@ + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + run-benchmarks + + exec + + + java + + -jar + ${project.build.directory}/benchmarks.jar + org.fastfilter.FilterBenchmark + -f + 1 + -wi + 1 + -i + 1 + + + + + diff --git a/jmh/src/main/java/org/fastfilter/ConstructionState.java b/jmh/src/main/java/org/fastfilter/ConstructionState.java index 4774f86..e061c80 100644 --- a/jmh/src/main/java/org/fastfilter/ConstructionState.java +++ b/jmh/src/main/java/org/fastfilter/ConstructionState.java @@ -19,8 +19,6 @@ public class ConstructionState { "BLOCKED_BLOOM", "SUCCINCT_COUNTING_BLOCKED_BLOOM", "SUCCINCT_COUNTING_BLOCKED_BLOOM_RANKED", - "XOR_SIMPLE", - "XOR_SIMPLE_2", "XOR_8", "XOR_16", "XOR_BINARY_FUSE_8", diff --git a/jmh/src/main/java/org/fastfilter/FilterBenchmark.java b/jmh/src/main/java/org/fastfilter/FilterBenchmark.java new file mode 100644 index 0000000..9f0be05 --- /dev/null +++ b/jmh/src/main/java/org/fastfilter/FilterBenchmark.java @@ -0,0 +1,130 @@ +package org.fastfilter; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import org.fastfilter.Filter; +import org.fastfilter.xor.Xor8; +import org.fastfilter.xor.Xor16; +import org.fastfilter.xor.XorBinaryFuse8; +import org.fastfilter.xor.XorBinaryFuse16; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +@State(Scope.Benchmark) +public class FilterBenchmark { + + @Param({"XOR_8", "XOR_16", "XOR_BINARY_FUSE_8", "XOR_BINARY_FUSE_16"}) + public String filterType; + + private Filter filter; + private long[] testKeys; + private final int NUM_KEYS = 1_000_000; + + @Setup + public void setup() { + // Create 1,000,000 keys (even numbers) + testKeys = new long[NUM_KEYS]; + for (int i = 0; i < testKeys.length; i++) { + testKeys[i] = (long) i * 2L; // even numbers + } + + try { + switch (filterType) { + case "XOR_8": + filter = Xor8.construct(testKeys); + break; + case "XOR_16": + filter = Xor16.construct(testKeys); + break; + case "XOR_BINARY_FUSE_8": + filter = XorBinaryFuse8.construct(testKeys); + break; + case "XOR_BINARY_FUSE_16": + filter = XorBinaryFuse16.construct(testKeys); + break; + default: + throw new IllegalArgumentException("Unknown filter type: " + filterType); + } + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @TearDown + public void tearDown() { + filter = null; + testKeys = null; + } + + @Benchmark + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsExisting(Blackhole blackhole) throws Throwable { + for (long key : testKeys) { + if (!filter.mayContain(key)) { + throw new RuntimeException("Key should exist: " + key); + } + } + } + + @Benchmark + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsNonExisting(Blackhole blackhole) throws Throwable { + int fp = 0; + for (int i = 0; i < testKeys.length; i++) { + long key = (long) i * 2L + 1L; // odd numbers + if (filter.mayContain(key)) { + fp++; + } + } + if (fp > 10000) { + throw new RuntimeException("Too many false positives: " + fp); + } + } + + @Benchmark + @BenchmarkMode(Mode.Throughput) + @OutputTimeUnit(TimeUnit.SECONDS) + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsExistingThroughput(Blackhole blackhole) throws Throwable { + for (long key : testKeys) { + if (!filter.mayContain(key)) { + throw new RuntimeException("Key should exist: " + key); + } + } + } + + @Benchmark + @BenchmarkMode(Mode.Throughput) + @OutputTimeUnit(TimeUnit.SECONDS) + @OperationsPerInvocation(NUM_KEYS) + public void benchmarkContainsNonExistingThroughput(Blackhole blackhole) throws Throwable { + int fp = 0; + for (int i = 0; i < testKeys.length; i++) { + long key = (long) i * 2L + 1L; // odd numbers + if (filter.mayContain(key)) { + fp++; + } + } + if (fp > 10000) { + throw new RuntimeException("Too many false positives: " + fp); + } + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(FilterBenchmark.class.getSimpleName()) + .build(); + + new Runner(opt).run(); + } +} diff --git a/pom.xml b/pom.xml index 101feba..727c8f2 100644 --- a/pom.xml +++ b/pom.xml @@ -1,13 +1,11 @@ - + 4.0.0 io.github.fastfilter fastfilter_java pom - 1.0.3-SNAPSHOT + 1.0.6-SNAPSHOT fastfilter jmh @@ -46,19 +44,20 @@ - scm:git:git://git@github.com:FastFilter/fastfilter_java.git - scm:git:ssh://git@github.com:FastFilter/fastfilter_java.git + scm:git:git@github.com:FastFilter/fastfilter_java.git + scm:git:git@github.com:FastFilter/fastfilter_java.git https://github.com/FastFilter/fastfilter_java/tree/master - + HEAD + ossrh - https://s01.oss.sonatype.org/content/repositories/snapshots + https://ossrh-staging-api.central.sonatype.com/content/repositories/snapshots ossrh - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + https://ossrh-staging-api.central.sonatype.com/service/local/staging/deploy/maven2/