From 8414324f38e1a9cfc680c4934fee06cd2e2535a9 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 22 Sep 2016 12:55:11 -0400 Subject: [PATCH 01/27] Update pom.xml --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 65d6754..8f10e4f 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ com/kamikaze/pfordelta/* me/lemire/integercompression/benchmarktools/* + ../../test/java/* From 26ac83afb1a804fe6ce2eef105e1123531ba1367 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 22 Sep 2016 13:01:42 -0400 Subject: [PATCH 02/27] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f10e4f..ae72409 100644 --- a/pom.xml +++ b/pom.xml @@ -134,7 +134,7 @@ com/kamikaze/pfordelta/* me/lemire/integercompression/benchmarktools/* - ../../test/java/* + ../../../test/java/* From 67d6e78530339e90445d3a06e4c0490a275ac6a7 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 13:14:36 -0500 Subject: [PATCH 03/27] Add files via upload --- .../integercompression/XueAddBasicTest.java | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 src/test/java/me/lemire/integercompression/XueAddBasicTest.java diff --git a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java new file mode 100644 index 0000000..e8ba042 --- /dev/null +++ b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java @@ -0,0 +1,135 @@ +package me.lemire.integercompression; + +import org.junit.Test; + +import java.util.Arrays; + + +/** + * Just some basic sanity tests. + * + * @author Daniel Lemire + */ +@SuppressWarnings({ "static-method" }) +public class XueAddBasicTest { + SkippableIntegerCODEC[] codecs = { + new GroupSimple9() + }; + + + /** + * + */ + @Test + public void consistentTest() { + int N = 4096; + int[] data = new int[N]; + int[] rev = new int[N]; + for (int k = 0; k < N; ++k) + data[k] = k % 128; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.consistentTest] codec = " + + c); + int[] outBuf = new int[N + 1024]; + for (int n = 0; n <= N; ++n) { + IntWrapper inPos = new IntWrapper(); + IntWrapper outPos = new IntWrapper(); + c.headlessCompress(data, inPos, n, outBuf, outPos); + + IntWrapper inPoso = new IntWrapper(); + IntWrapper outPoso = new IntWrapper(); + c.headlessUncompress(outBuf, inPoso, outPos.get(), rev, + outPoso, n); + if (outPoso.get() != n) { + throw new RuntimeException("bug "+n); + } + if (inPoso.get() != outPos.get()) { + throw new RuntimeException("bug "+n+" "+inPoso.get()+" "+outPos.get()); + } + for (int j = 0; j < n; ++j) + if (data[j] != rev[j]) { + throw new RuntimeException("bug"); + } + } + } + } + + + /** + * + */ + @Test + public void varyingLengthTest() { + int N = 4096; + int[] data = new int[N]; + for (int k = 0; k < N; ++k) + data[k] = k; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.varyingLengthTest] codec = "+c); + for (int L = 1; L <= 128; L++) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug "+c.toString()+" "+k+" "+answer[k]+" "+data[k]); + } + for (int L = 128; L <= N; L *= 2) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug"); + } + + } + } + + /** + * + */ + /* + @Test + + public void varyingLengthTest2() { + int N = 128; + int[] data = new int[N]; + data[127] = -1; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.varyingLengthTest2] codec = "+c); + + try { + // CODEC Simple9 is limited to "small" integers. + if (c.getClass().equals( + Class.forName("me.lemire.integercompression.Simple9"))) + continue; + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + try { + // CODEC Simple16 is limited to "small" integers. + if (c.getClass().equals( + Class.forName("me.lemire.integercompression.Simple16"))) + continue; + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + for (int L = 1; L <= 128; L++) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug at k = "+k+" "+answer[k]+" "+data[k]+" for "+c.toString()); + } + for (int L = 128; L <= N; L *= 2) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug"); + } + + } + } + + */ +} From f9bb5a9b4426f2979a6e83582213c5c04d4d0402 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 16:22:29 -0500 Subject: [PATCH 04/27] Update XueAddBasicTest.java --- src/test/java/me/lemire/integercompression/XueAddBasicTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java index e8ba042..03775a0 100644 --- a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java +++ b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java @@ -22,7 +22,7 @@ public class XueAddBasicTest { */ @Test public void consistentTest() { - int N = 4096; + int N = 100; int[] data = new int[N]; int[] rev = new int[N]; for (int k = 0; k < N; ++k) From fc09276e2fabd662ed73569a6da808d0cdf37131 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 16:29:57 -0500 Subject: [PATCH 05/27] Update XueAddBasicTest.java --- .../java/me/lemire/integercompression/XueAddBasicTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java index 03775a0..ed7a612 100644 --- a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java +++ b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java @@ -22,11 +22,11 @@ public class XueAddBasicTest { */ @Test public void consistentTest() { - int N = 100; + int N = 4096; int[] data = new int[N]; int[] rev = new int[N]; for (int k = 0; k < N; ++k) - data[k] = k % 128; + data[k] = k % 100; for (SkippableIntegerCODEC c : codecs) { System.out.println("[SkippeableBasicTest.consistentTest] codec = " + c); From 18850c1f07704f55215098abb1cecd81df0b4f27 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 16:33:04 -0500 Subject: [PATCH 06/27] Update XueAddBasicTest.java --- .../java/me/lemire/integercompression/XueAddBasicTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java index ed7a612..8c08ebb 100644 --- a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java +++ b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java @@ -25,8 +25,9 @@ public void consistentTest() { int N = 4096; int[] data = new int[N]; int[] rev = new int[N]; + for (int mod = 1; mod <= 81; ++mod){ for (int k = 0; k < N; ++k) - data[k] = k % 100; + data[k] = k % mod; for (SkippableIntegerCODEC c : codecs) { System.out.println("[SkippeableBasicTest.consistentTest] codec = " + c); @@ -52,6 +53,7 @@ public void consistentTest() { } } } + } } From f294120754f2d515c08038de7e10553e7a7f3f06 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 18:05:43 -0500 Subject: [PATCH 07/27] Add files via upload --- .../integercompression/XueAddBasicTest.java | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java index 8c08ebb..240e5b8 100644 --- a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java +++ b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java @@ -22,12 +22,38 @@ public class XueAddBasicTest { */ @Test public void consistentTest() { - int N = 4096; - int[] data = new int[N]; + int bitLength[] = { 1, 2, 3, 4, 5, 7, 9, 14, 28 }; + int codeNum[] = { 28, 14, 9, 7, 5, 4, 3, 2, 1 }; + for(int s1 = 0 ; s1 < 9; ++s1){ + for(int s2 = 0; s2 < 9; ++s2){ + int max1 = 1 << bitLength[s1]; + int max2 = 1 << bitLength[s2]; + int num1 = codeNum[s1]; + int num2 = codeNum[s2]; + + int N = num1 + num2; + int[] data = new int[N]; + + int i = 0; + for(; i < num1; ++i){ + data[i] = max1 - 1; + } + for(; i < num1 + num2; ++i){ + data[i] = max2 - 1; + } + + oneTest(N, data); + + } + } + + + } + private void oneTest(int N, int[] data){ int[] rev = new int[N]; - for (int mod = 1; mod <= 81; ++mod){ - for (int k = 0; k < N; ++k) - data[k] = k % mod; + + //for (int k = 0; k < N; ++k) + // data[k] = k % 128; for (SkippableIntegerCODEC c : codecs) { System.out.println("[SkippeableBasicTest.consistentTest] codec = " + c); @@ -53,9 +79,7 @@ public void consistentTest() { } } } - } } - /** * From 09584a326b7c25d13a8a5ada0793bb9855e82489 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 19:13:54 -0500 Subject: [PATCH 08/27] Add files via upload --- .../java/me/lemire/integercompression/XueAddBasicTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java index 240e5b8..5cd3675 100644 --- a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java +++ b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java @@ -32,13 +32,14 @@ public void consistentTest() { int num2 = codeNum[s2]; int N = num1 + num2; + if(N < 57) N = 57; int[] data = new int[N]; int i = 0; for(; i < num1; ++i){ data[i] = max1 - 1; } - for(; i < num1 + num2; ++i){ + for(; i < N; ++i){ data[i] = max2 - 1; } From cb6d2ded4f457684436d295207cfe95be1ca3488 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 19:48:57 -0500 Subject: [PATCH 09/27] Update BasicTest.java --- .../lemire/integercompression/BasicTest.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/test/java/me/lemire/integercompression/BasicTest.java b/src/test/java/me/lemire/integercompression/BasicTest.java index cbdb60e..e5ffb30 100644 --- a/src/test/java/me/lemire/integercompression/BasicTest.java +++ b/src/test/java/me/lemire/integercompression/BasicTest.java @@ -245,10 +245,15 @@ public void verifyBitPacking() { int[] data = new int[N]; int[] compressed = new int[N]; int[] uncompressed = new int[N]; - for (int bit = 0; bit < 31; ++bit) { + for (int bit = 0; bit < 32; ++bit) {//Xue for (int t = 0; t < TIMES; ++t) { for (int k = 0; k < N; ++k) { - data[k] = r.nextInt(1 << bit); + if(bit == 31){ + data[k] = r.nextInt(1 << 30); + } + else { + data[k] = r.nextInt(1 << bit); + } } BitPacking.fastpack(data, 0, compressed, 0, bit); BitPacking.fastunpack(compressed, 0, uncompressed, 0, bit); @@ -268,10 +273,15 @@ public void verifyWithoutMask() { int[] data = new int[N]; int[] compressed = new int[N]; int[] uncompressed = new int[N]; - for (int bit = 0; bit < 31; ++bit) { + for (int bit = 0; bit < 32; ++bit) {//Xue for (int t = 0; t < TIMES; ++t) { for (int k = 0; k < N; ++k) { - data[k] = r.nextInt(1 << bit); + if(bit == 31){ + data[k] = r.nextInt(1 << 30); + } + else { + data[k] = r.nextInt(1 << bit); + } } BitPacking.fastpackwithoutmask(data, 0, compressed, 0, bit); BitPacking.fastunpack(compressed, 0, uncompressed, 0, bit); From c7d943ac8e992a29a8aefb917cae1a8421e08c87 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 20:16:54 -0500 Subject: [PATCH 10/27] Delete XueAddBasicTest.java --- .../integercompression/XueAddBasicTest.java | 162 ------------------ 1 file changed, 162 deletions(-) delete mode 100644 src/test/java/me/lemire/integercompression/XueAddBasicTest.java diff --git a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java b/src/test/java/me/lemire/integercompression/XueAddBasicTest.java deleted file mode 100644 index 5cd3675..0000000 --- a/src/test/java/me/lemire/integercompression/XueAddBasicTest.java +++ /dev/null @@ -1,162 +0,0 @@ -package me.lemire.integercompression; - -import org.junit.Test; - -import java.util.Arrays; - - -/** - * Just some basic sanity tests. - * - * @author Daniel Lemire - */ -@SuppressWarnings({ "static-method" }) -public class XueAddBasicTest { - SkippableIntegerCODEC[] codecs = { - new GroupSimple9() - }; - - - /** - * - */ - @Test - public void consistentTest() { - int bitLength[] = { 1, 2, 3, 4, 5, 7, 9, 14, 28 }; - int codeNum[] = { 28, 14, 9, 7, 5, 4, 3, 2, 1 }; - for(int s1 = 0 ; s1 < 9; ++s1){ - for(int s2 = 0; s2 < 9; ++s2){ - int max1 = 1 << bitLength[s1]; - int max2 = 1 << bitLength[s2]; - int num1 = codeNum[s1]; - int num2 = codeNum[s2]; - - int N = num1 + num2; - if(N < 57) N = 57; - int[] data = new int[N]; - - int i = 0; - for(; i < num1; ++i){ - data[i] = max1 - 1; - } - for(; i < N; ++i){ - data[i] = max2 - 1; - } - - oneTest(N, data); - - } - } - - - } - private void oneTest(int N, int[] data){ - int[] rev = new int[N]; - - //for (int k = 0; k < N; ++k) - // data[k] = k % 128; - for (SkippableIntegerCODEC c : codecs) { - System.out.println("[SkippeableBasicTest.consistentTest] codec = " - + c); - int[] outBuf = new int[N + 1024]; - for (int n = 0; n <= N; ++n) { - IntWrapper inPos = new IntWrapper(); - IntWrapper outPos = new IntWrapper(); - c.headlessCompress(data, inPos, n, outBuf, outPos); - - IntWrapper inPoso = new IntWrapper(); - IntWrapper outPoso = new IntWrapper(); - c.headlessUncompress(outBuf, inPoso, outPos.get(), rev, - outPoso, n); - if (outPoso.get() != n) { - throw new RuntimeException("bug "+n); - } - if (inPoso.get() != outPos.get()) { - throw new RuntimeException("bug "+n+" "+inPoso.get()+" "+outPos.get()); - } - for (int j = 0; j < n; ++j) - if (data[j] != rev[j]) { - throw new RuntimeException("bug"); - } - } - } - } - - /** - * - */ - @Test - public void varyingLengthTest() { - int N = 4096; - int[] data = new int[N]; - for (int k = 0; k < N; ++k) - data[k] = k; - for (SkippableIntegerCODEC c : codecs) { - System.out.println("[SkippeableBasicTest.varyingLengthTest] codec = "+c); - for (int L = 1; L <= 128; L++) { - int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); - int[] answer = TestUtils.uncompressHeadless(c, comp, L); - for (int k = 0; k < L; ++k) - if (answer[k] != data[k]) - throw new RuntimeException("bug "+c.toString()+" "+k+" "+answer[k]+" "+data[k]); - } - for (int L = 128; L <= N; L *= 2) { - int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); - int[] answer = TestUtils.uncompressHeadless(c, comp, L); - for (int k = 0; k < L; ++k) - if (answer[k] != data[k]) - throw new RuntimeException("bug"); - } - - } - } - - /** - * - */ - /* - @Test - - public void varyingLengthTest2() { - int N = 128; - int[] data = new int[N]; - data[127] = -1; - for (SkippableIntegerCODEC c : codecs) { - System.out.println("[SkippeableBasicTest.varyingLengthTest2] codec = "+c); - - try { - // CODEC Simple9 is limited to "small" integers. - if (c.getClass().equals( - Class.forName("me.lemire.integercompression.Simple9"))) - continue; - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - try { - // CODEC Simple16 is limited to "small" integers. - if (c.getClass().equals( - Class.forName("me.lemire.integercompression.Simple16"))) - continue; - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - for (int L = 1; L <= 128; L++) { - int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); - int[] answer = TestUtils.uncompressHeadless(c, comp, L); - for (int k = 0; k < L; ++k) - if (answer[k] != data[k]) - throw new RuntimeException("bug at k = "+k+" "+answer[k]+" "+data[k]+" for "+c.toString()); - } - for (int L = 128; L <= N; L *= 2) { - int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); - int[] answer = TestUtils.uncompressHeadless(c, comp, L); - for (int k = 0; k < L; ++k) - if (answer[k] != data[k]) - throw new RuntimeException("bug"); - } - - } - } - - */ -} From 682de73f21c07ebe133fb6c7b9eacb514b93d6aa Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 20:17:19 -0500 Subject: [PATCH 11/27] Add files via upload --- .../Xue_BenchmarkBitPackingTesting.java | 154 +++++++++++++++++ .../Xue_GroupSimpleTest.java | 162 ++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java create mode 100644 src/test/java/me/lemire/integercompression/Xue_GroupSimpleTest.java diff --git a/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java new file mode 100644 index 0000000..9e3343e --- /dev/null +++ b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java @@ -0,0 +1,154 @@ +package me.lemire.integercompression; /** + * This code is released under the + * Apache License Version 2.0 http://www.apache.org/licenses/. + * + * (c) Daniel Lemire, http://lemire.me/en/ + */ +//package me.lemire.integercompression.benchmarktools; +import me.lemire.integercompression.benchmarktools.*; +import me.lemire.integercompression.BitPacking; +import me.lemire.integercompression.differential.Delta; +import me.lemire.integercompression.differential.IntegratedBitPacking; + +import java.text.DecimalFormat; +import java.util.Arrays; +import java.util.Random; + +/** + * Class used to benchmark the speed of bit packing. (For expert use.) + * + * @author Daniel Lemire + * + */ +public class Xue_BenchmarkBitPackingTesting { + + private static void test(boolean verbose) { + DecimalFormat dfspeed = new DecimalFormat("0"); + final int N = 32; + final int times = 100000; + Random r = new Random(0); + int[] data = new int[N]; + int[] compressed = new int[N]; + int[] uncompressed = new int[N]; + for (int bit = 0; bit < 32; ++bit) { + long comp = 0; + long compwm = 0; + long decomp = 0; + for (int t = 0; t < times; ++t) { + for (int k = 0; k < N; ++k) { + data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)); + } + long time1 = System.nanoTime(); + BitPacking + .fastpack(data, 0, compressed, 0, bit); + long time2 = System.nanoTime(); + BitPacking.fastpackwithoutmask(data, 0, + compressed, 0, bit); + long time3 = System.nanoTime(); + BitPacking.fastunpack(compressed, 0, + uncompressed, 0, bit); + long time4 = System.nanoTime(); + comp += time2 - time1; + compwm += time3 - time2; + decomp += time4 - time3; + } + if (verbose) + System.out.println("bit = " + + bit + + " comp. speed = " + + dfspeed.format(N * times * 1000.0 + / (comp)) + + " comp. speed wm = " + + dfspeed.format(N * times * 1000.0 + / (compwm)) + + " decomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (decomp))); + } + } + + private static void testWithDeltas(boolean verbose) { + DecimalFormat dfspeed = new DecimalFormat("0"); + final int N = 32; + final int times = 100000; + Random r = new Random(0); + int[] data = new int[N]; + int[] compressed = new int[N]; + int[] icompressed = new int[N]; + int[] uncompressed = new int[N]; + for (int bit = 1; bit < 32; ++bit) { + long comp = 0; + long decomp = 0; + long icomp = 0; + long idecomp = 0; + for (int t = 0; t < times; ++t) { + data[0] = r.nextInt(1 << (bit == 31 ? 30 : bit)); + for (int k = 1; k < N; ++k) { + data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)) + + data[k - 1]; + } + int[] tmpdata = Arrays + .copyOf(data, data.length); + long time1 = System.nanoTime(); + Delta.delta(tmpdata); + BitPacking.fastpackwithoutmask(tmpdata, 0, + compressed, 0, bit); + long time2 = System.nanoTime(); + BitPacking.fastunpack(compressed, 0, + uncompressed, 0, bit); + Delta.fastinverseDelta(uncompressed); + long time3 = System.nanoTime(); + if (!Arrays.equals(data, uncompressed)) + throw new RuntimeException("bug"); + comp += time2 - time1; + decomp += time3 - time2; + tmpdata = Arrays.copyOf(data, data.length); + time1 = System.nanoTime(); + IntegratedBitPacking.integratedpack(0, tmpdata, + 0, icompressed, 0, bit); + time2 = System.nanoTime(); + IntegratedBitPacking.integratedunpack(0, + icompressed, 0, uncompressed, 0, bit); + time3 = System.nanoTime(); + if (!Arrays.equals(icompressed, compressed)) + throw new RuntimeException("ibug " + + bit); + if (!Arrays.equals(data, uncompressed)) + throw new RuntimeException("bug " + bit); + icomp += time2 - time1; + idecomp += time3 - time2; + } + if (verbose) + System.out.println("bit = " + + bit + + " comp. speed = " + + dfspeed.format(N * times * 1000.0 + / (comp)) + + " decomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (decomp)) + + " icomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (icomp)) + + " idecomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (idecomp))); + } + } + + /** + * Main method + * + * @param args + * command-line arguments + */ + public static void main(String[] args) { + System.out.println("Testing packing and delta "); + testWithDeltas(false); + testWithDeltas(true); + System.out.println("Testing packing alone "); + test(false); + test(true); + } + +} diff --git a/src/test/java/me/lemire/integercompression/Xue_GroupSimpleTest.java b/src/test/java/me/lemire/integercompression/Xue_GroupSimpleTest.java new file mode 100644 index 0000000..8f76e90 --- /dev/null +++ b/src/test/java/me/lemire/integercompression/Xue_GroupSimpleTest.java @@ -0,0 +1,162 @@ +package me.lemire.integercompression; + +import org.junit.Test; + +import java.util.Arrays; + + +/** + * Just some basic sanity tests. + * + * @author Daniel Lemire + */ +@SuppressWarnings({ "static-method" }) +public class Xue_GroupSimpleTest { + SkippableIntegerCODEC[] codecs = { + new GroupSimple9() + }; + + + /** + * + */ + @Test + public void consistentTest() { + int bitLength[] = { 1, 2, 3, 4, 5, 7, 9, 14, 28 }; + int codeNum[] = { 28, 14, 9, 7, 5, 4, 3, 2, 1 }; + for(int s1 = 0 ; s1 < 9; ++s1){ + for(int s2 = 0; s2 < 9; ++s2){ + int max1 = 1 << bitLength[s1]; + int max2 = 1 << bitLength[s2]; + int num1 = codeNum[s1]; + int num2 = codeNum[s2]; + + int N = num1 + num2; + if(N < 57) N = 57; + int[] data = new int[N]; + + int i = 0; + for(; i < num1; ++i){ + data[i] = max1 - 1; + } + for(; i < N; ++i){ + data[i] = max2 - 1; + } + + oneTest(N, data); + + } + } + + + } + private void oneTest(int N, int[] data){ + int[] rev = new int[N]; + + //for (int k = 0; k < N; ++k) + // data[k] = k % 128; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.consistentTest] codec = " + + c); + int[] outBuf = new int[N + 1024]; + for (int n = 0; n <= N; ++n) { + IntWrapper inPos = new IntWrapper(); + IntWrapper outPos = new IntWrapper(); + c.headlessCompress(data, inPos, n, outBuf, outPos); + + IntWrapper inPoso = new IntWrapper(); + IntWrapper outPoso = new IntWrapper(); + c.headlessUncompress(outBuf, inPoso, outPos.get(), rev, + outPoso, n); + if (outPoso.get() != n) { + throw new RuntimeException("bug "+n); + } + if (inPoso.get() != outPos.get()) { + throw new RuntimeException("bug "+n+" "+inPoso.get()+" "+outPos.get()); + } + for (int j = 0; j < n; ++j) + if (data[j] != rev[j]) { + throw new RuntimeException("bug"); + } + } + } + } + + /** + * + */ + @Test + public void varyingLengthTest() { + int N = 4096; + int[] data = new int[N]; + for (int k = 0; k < N; ++k) + data[k] = k; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.varyingLengthTest] codec = "+c); + for (int L = 1; L <= 128; L++) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug "+c.toString()+" "+k+" "+answer[k]+" "+data[k]); + } + for (int L = 128; L <= N; L *= 2) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug"); + } + + } + } + + /** + * + */ + /* + @Test + + public void varyingLengthTest2() { + int N = 128; + int[] data = new int[N]; + data[127] = -1; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.varyingLengthTest2] codec = "+c); + + try { + // CODEC Simple9 is limited to "small" integers. + if (c.getClass().equals( + Class.forName("me.lemire.integercompression.Simple9"))) + continue; + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + try { + // CODEC Simple16 is limited to "small" integers. + if (c.getClass().equals( + Class.forName("me.lemire.integercompression.Simple16"))) + continue; + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + for (int L = 1; L <= 128; L++) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug at k = "+k+" "+answer[k]+" "+data[k]+" for "+c.toString()); + } + for (int L = 128; L <= N; L *= 2) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug"); + } + + } + } + + */ +} From d7adc43c6b68f3c07cd9457a33c4d84ecf3a289f Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 20:18:22 -0500 Subject: [PATCH 12/27] Add files via upload --- .../integercompression/Xue_BenchmarkBitPackingTesting.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java index 9e3343e..ae85845 100644 --- a/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java +++ b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java @@ -30,13 +30,13 @@ private static void test(boolean verbose) { int[] data = new int[N]; int[] compressed = new int[N]; int[] uncompressed = new int[N]; - for (int bit = 0; bit < 32; ++bit) { + for (int bit = 0; bit < 31; ++bit) { long comp = 0; long compwm = 0; long decomp = 0; for (int t = 0; t < times; ++t) { for (int k = 0; k < N; ++k) { - data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)); + data[k] = r.nextInt(1 << bit); } long time1 = System.nanoTime(); BitPacking From 20ef015c59275cdd112aca55f6e3fb8bd5e8f957 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 20:20:02 -0500 Subject: [PATCH 13/27] Add files via upload --- .../integercompression/Xue_BenchmarkBitPackingTesting.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java index ae85845..9e3343e 100644 --- a/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java +++ b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java @@ -30,13 +30,13 @@ private static void test(boolean verbose) { int[] data = new int[N]; int[] compressed = new int[N]; int[] uncompressed = new int[N]; - for (int bit = 0; bit < 31; ++bit) { + for (int bit = 0; bit < 32; ++bit) { long comp = 0; long compwm = 0; long decomp = 0; for (int t = 0; t < times; ++t) { for (int k = 0; k < N; ++k) { - data[k] = r.nextInt(1 << bit); + data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)); } long time1 = System.nanoTime(); BitPacking From 15d05548d8334b03cb992ae8adc97bcd016a699f Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 20:47:49 -0500 Subject: [PATCH 14/27] Add files via upload --- .../Xue_BenchmarkBitPackingTesting.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java index 9e3343e..9c7b8b9 100644 --- a/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java +++ b/src/test/java/me/lemire/integercompression/Xue_BenchmarkBitPackingTesting.java @@ -10,6 +10,7 @@ import me.lemire.integercompression.differential.Delta; import me.lemire.integercompression.differential.IntegratedBitPacking; +import org.junit.Test; import java.text.DecimalFormat; import java.util.Arrays; import java.util.Random; @@ -17,12 +18,16 @@ /** * Class used to benchmark the speed of bit packing. (For expert use.) * - * @author Daniel Lemire + * @author Xue Li * */ public class Xue_BenchmarkBitPackingTesting { - - private static void test(boolean verbose) { + @Test + public void test(){ + testCore(true); + testCore(false); + } + private void testCore(boolean verbose) { DecimalFormat dfspeed = new DecimalFormat("0"); final int N = 32; final int times = 100000; @@ -66,8 +71,12 @@ private static void test(boolean verbose) { / (decomp))); } } - - private static void testWithDeltas(boolean verbose) { + @Test + public void testWithDeltas(){ + testWithDeltasCore(false); + testWithDeltasCore(true); + } + private void testWithDeltasCore(boolean verbose) { DecimalFormat dfspeed = new DecimalFormat("0"); final int N = 32; final int times = 100000; @@ -104,6 +113,9 @@ private static void testWithDeltas(boolean verbose) { decomp += time3 - time2; tmpdata = Arrays.copyOf(data, data.length); time1 = System.nanoTime(); + if(bit == 26){ + int tttt = 0; + } IntegratedBitPacking.integratedpack(0, tmpdata, 0, icompressed, 0, bit); time2 = System.nanoTime(); @@ -142,6 +154,7 @@ private static void testWithDeltas(boolean verbose) { * @param args * command-line arguments */ + /* public static void main(String[] args) { System.out.println("Testing packing and delta "); testWithDeltas(false); @@ -150,5 +163,6 @@ public static void main(String[] args) { test(false); test(true); } + */ } From 2c9307251e27879ffcc9768ade021299aae87a17 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 20:57:05 -0500 Subject: [PATCH 15/27] Add files via upload --- .../Xue_IntegratedBitPackingTest.java | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/test/java/me/lemire/integercompression/Xue_IntegratedBitPackingTest.java diff --git a/src/test/java/me/lemire/integercompression/Xue_IntegratedBitPackingTest.java b/src/test/java/me/lemire/integercompression/Xue_IntegratedBitPackingTest.java new file mode 100644 index 0000000..30a9a63 --- /dev/null +++ b/src/test/java/me/lemire/integercompression/Xue_IntegratedBitPackingTest.java @@ -0,0 +1,152 @@ +package me.lemire.integercompression; + +import me.lemire.integercompression.differential.Delta; +import me.lemire.integercompression.differential.IntegratedBitPacking; +import org.junit.Test; + +import java.text.DecimalFormat; +import java.util.Arrays; +import java.util.Random; + + +/** + * Tests for class GroupSimple9. + * + * @author Xue Li + */ +@SuppressWarnings({ "static-method" }) +public class Xue_IntegratedBitPackingTest { + + + /** + * + */ + @Test + public void Test() { + testCore(true); + } + private void testCore(boolean verbose) { + DecimalFormat dfspeed = new DecimalFormat("0"); + final int N = 32; + final int times = 100000; + Random r = new Random(0); + int[] data = new int[N]; + int[] compressed = new int[N]; + int[] uncompressed = new int[N]; + for (int bit = 0; bit < 32; ++bit) { + long comp = 0; + long compwm = 0; + long decomp = 0; + for (int t = 0; t < times; ++t) { + for (int k = 0; k < N; ++k) { + data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)); + } + long time1 = System.nanoTime(); + BitPacking + .fastpack(data, 0, compressed, 0, bit); + long time2 = System.nanoTime(); + BitPacking.fastpackwithoutmask(data, 0, + compressed, 0, bit); + long time3 = System.nanoTime(); + BitPacking.fastunpack(compressed, 0, + uncompressed, 0, bit); + long time4 = System.nanoTime(); + comp += time2 - time1; + compwm += time3 - time2; + decomp += time4 - time3; + } + if (verbose) + System.out.println("bit = " + + bit + + " comp. speed = " + + dfspeed.format(N * times * 1000.0 + / (comp)) + + " comp. speed wm = " + + dfspeed.format(N * times * 1000.0 + / (compwm)) + + " decomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (decomp))); + } + } + + /** + * + */ + @Test + public void varyingLengthTest() { + testWithDeltasCore(true); + } + + private void testWithDeltasCore(boolean verbose) { + DecimalFormat dfspeed = new DecimalFormat("0"); + final int N = 32; + final int times = 100000; + Random r = new Random(0); + int[] data = new int[N]; + int[] compressed = new int[N]; + int[] icompressed = new int[N]; + int[] uncompressed = new int[N]; + for (int bit = 1; bit < 32; ++bit) { + long comp = 0; + long decomp = 0; + long icomp = 0; + long idecomp = 0; + for (int t = 0; t < times; ++t) { + data[0] = r.nextInt(1 << (bit == 31 ? 30 : bit)); + for (int k = 1; k < N; ++k) { + data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)) + + data[k - 1]; + } + int[] tmpdata = Arrays + .copyOf(data, data.length); + long time1 = System.nanoTime(); + Delta.delta(tmpdata); + BitPacking.fastpackwithoutmask(tmpdata, 0, + compressed, 0, bit); + long time2 = System.nanoTime(); + BitPacking.fastunpack(compressed, 0, + uncompressed, 0, bit); + Delta.fastinverseDelta(uncompressed); + long time3 = System.nanoTime(); + if (!Arrays.equals(data, uncompressed)) + throw new RuntimeException("bug"); + comp += time2 - time1; + decomp += time3 - time2; + tmpdata = Arrays.copyOf(data, data.length); + time1 = System.nanoTime(); + if(bit == 26){ + int tttt = 0; + } + IntegratedBitPacking.integratedpack(0, tmpdata, + 0, icompressed, 0, bit); + time2 = System.nanoTime(); + IntegratedBitPacking.integratedunpack(0, + icompressed, 0, uncompressed, 0, bit); + time3 = System.nanoTime(); + if (!Arrays.equals(icompressed, compressed)) + throw new RuntimeException("ibug " + + bit); + if (!Arrays.equals(data, uncompressed)) + throw new RuntimeException("bug " + bit); + icomp += time2 - time1; + idecomp += time3 - time2; + } + if (verbose) + System.out.println("bit = " + + bit + + " comp. speed = " + + dfspeed.format(N * times * 1000.0 + / (comp)) + + " decomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (decomp)) + + " icomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (icomp)) + + " idecomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (idecomp))); + } + } +} From 46bd95b5dfc1abf2b6ebbbe8ba96f988affac080 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 22:29:35 -0500 Subject: [PATCH 16/27] Add files via upload --- .../integercompression/Xue_DeltaTest.java | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 src/test/java/me/lemire/integercompression/Xue_DeltaTest.java diff --git a/src/test/java/me/lemire/integercompression/Xue_DeltaTest.java b/src/test/java/me/lemire/integercompression/Xue_DeltaTest.java new file mode 100644 index 0000000..910fc5d --- /dev/null +++ b/src/test/java/me/lemire/integercompression/Xue_DeltaTest.java @@ -0,0 +1,97 @@ +package me.lemire.integercompression; + +import org.junit.Test; + +import java.util.Arrays; +import me.lemire.integercompression.differential.Delta; + +import static org.junit.Assert.assertEquals; + + +/** + * Tests for class Delta. + * + * @author Xue Li + */ +@SuppressWarnings({ "static-method" }) +public class Xue_DeltaTest { + + /** + * + */ + @Test + public void simpleDeltaTest() { + + int[] data = new int[1024]; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + Delta.delta(data); + + for (int k = 1; k < 1024; ++k){ + assertEquals("simple delta", data[k], 1); + } + } + + /** + * + */ + @Test + public void deltaWithInitialTest() { + int[] data = new int[1024]; //0 1 2 3 4 5 6 7 8 9 10 ... => 0 1 2 -5 1 1 1 1 8 9 10, return 7 + int start = 3, length = 5, init = 8; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + Delta.delta(data, start, length, init); + + for (int k = 0; k < 1024; ++k){ + if(k == start) assertEquals("deltaWithInitialTest", data[k], start - init); + else if(k > start && k < start + length) assertEquals("deltaWithInitialTest", data[k], 1); + else assertEquals("deltaWithInitialTest", data[k], k); + } + } + + @Test + public void inverseDeltaTest(){ + int[] data = new int[1024]; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + Delta.delta(data); + Delta.inverseDelta(data); + + for (int k = 0; k < 1024; ++k){ + assertEquals("inverseDeltaTest", data[k], k); + } + } + + @Test + public void fastinverseDeltaTest(){ + int[] data = new int[1024]; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + Delta.delta(data); + Delta.fastinverseDelta(data); + + for (int k = 0; k < 1024; ++k){ + assertEquals("inverseDeltaTest", data[k], k); + } + } + + @Test + public void fastinverseDeltaWithInitialTest(){ + int[] data = new int[1024]; //0 1 2 3 4 5 6 7 8 9 10 ... => 0 1 2 -5 1 1 1 1 8 9 10, return 7 + int start = 3, length = 5, init = 8; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + int newInit = Delta.delta(data, start, length, init); + Delta.fastinverseDelta(data, start, length, init); + + for (int k = 0; k < 1024; ++k){ + assertEquals("inverseDeltaTest", data[k], k); + } + } +} From 42d472c36518b1bfd00a99828b0643235bde9f09 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 1 Dec 2016 22:55:11 -0500 Subject: [PATCH 17/27] Add files via upload --- .../integercompression/Xue_DeltaTest.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/test/java/me/lemire/integercompression/Xue_DeltaTest.java b/src/test/java/me/lemire/integercompression/Xue_DeltaTest.java index 910fc5d..623ef67 100644 --- a/src/test/java/me/lemire/integercompression/Xue_DeltaTest.java +++ b/src/test/java/me/lemire/integercompression/Xue_DeltaTest.java @@ -39,16 +39,17 @@ public void simpleDeltaTest() { @Test public void deltaWithInitialTest() { int[] data = new int[1024]; //0 1 2 3 4 5 6 7 8 9 10 ... => 0 1 2 -5 1 1 1 1 8 9 10, return 7 - int start = 3, length = 5, init = 8; + + int start = 3, length = 5, init; + int[] out = new int[length]; for (int k = 0; k < 1024; ++k) data[k] = k; + init = data[start - 1]; - Delta.delta(data, start, length, init); + Delta.delta(data, start, length, init, out); - for (int k = 0; k < 1024; ++k){ - if(k == start) assertEquals("deltaWithInitialTest", data[k], start - init); - else if(k > start && k < start + length) assertEquals("deltaWithInitialTest", data[k], 1); - else assertEquals("deltaWithInitialTest", data[k], k); + for (int k = 0; k < length; ++k){ + assertEquals("deltaWithInitialTest", out[k], 1); } } @@ -82,16 +83,13 @@ public void fastinverseDeltaTest(){ @Test public void fastinverseDeltaWithInitialTest(){ - int[] data = new int[1024]; //0 1 2 3 4 5 6 7 8 9 10 ... => 0 1 2 -5 1 1 1 1 8 9 10, return 7 - int start = 3, length = 5, init = 8; + int[] data = {0, 1, 2, 1, 1, 1, 1, 1, 8, 9}; //0 1 2 3 4 5 6 7 8 9 10 ... => 0 1 2 -5 1 1 1 1 8 9 10, return 7 + int start = 3, length = 5, init = 2; - for (int k = 0; k < 1024; ++k) data[k] = k; - - int newInit = Delta.delta(data, start, length, init); Delta.fastinverseDelta(data, start, length, init); - for (int k = 0; k < 1024; ++k){ - assertEquals("inverseDeltaTest", data[k], k); + for (int k = 0; k < length; ++k){ + assertEquals("fastinverseDeltaWithInitialTest", data[k], k); } } } From 64eea0e21c70582dc0e7aa489e3bacea37fc8716 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Thu, 8 Dec 2016 17:53:47 -0500 Subject: [PATCH 18/27] Add files via upload --- .../pfordelta/Simple16WithHardCodesTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java diff --git a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java new file mode 100644 index 0000000..0ba1d33 --- /dev/null +++ b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java @@ -0,0 +1,23 @@ +package com.kamikaze.pfordelta; + +import org.junit.Test; +import com.kamikaze.*; + +import static org.junit.Assert.assertEquals; + +/** + * Created by xueli on 12/8/16. + */ +public class Simple16WithHardCodesTest { + Simple16WithHardCodes sm = new Simple16WithHardCodes(); + + @Test + public void test(){ + int[] input = new int[1024]; + int[] output = new int[1024]; + + for (int k = 0; k < 1024; ++k) input[k] = k; + int k = sm.s16Compress(output, 0, input, 0, 1024, 10, 10, input); + assertEquals(k, 8); + } +} From 3f10fe4cb5df9a182fb10939696c45caf312b3f9 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 10:01:30 -0500 Subject: [PATCH 19/27] Update pom.xml --- pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pom.xml b/pom.xml index ae72409..0d018d4 100644 --- a/pom.xml +++ b/pom.xml @@ -132,8 +132,6 @@ 0.7.6.201602180812 - com/kamikaze/pfordelta/* - me/lemire/integercompression/benchmarktools/* ../../../test/java/* From 39ab914d4dce776533a051b77dbbe61beb230758 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 10:21:01 -0500 Subject: [PATCH 20/27] Add files via upload --- .../pfordelta/Simple16WithHardCodesTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java index 0ba1d33..1635288 100644 --- a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java +++ b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java @@ -19,5 +19,26 @@ public void test(){ for (int k = 0; k < 1024; ++k) input[k] = k; int k = sm.s16Compress(output, 0, input, 0, 1024, 10, 10, input); assertEquals(k, 8); + + assertEquals(sm.s16CompressBackup(output, 0, input, 0, 1024, 10, 10, input), 8); + + assertEquals(sm.s16Decompress(output, 0, input, 0, 1024), 28); + + assertEquals(sm.s16DecompressWithIntBufferBackup(output, 0, 22, 1024), 28); + + assertEquals(sm.s16DecompressWithIntBuffer(output, 0, 22, 1024), 28); + + assertEquals(sm.s16DecompressWithIntBufferWithHardCodes(output, 0, 22, 1024), 28); + + assertEquals(sm.s16DecompressWithIntBufferIntegrated(output, 0, 22, 1024, input, 8), 28); + + assertEquals(sm.s16DecompressWithIntBufferIntegrated2(output, 0, 22, 1024, input, 8), 28); + + assertEquals(sm.s16DecompressWithIntBufferIntegratedBackup(output, 0, 22, 1024, input, 8), 28); + + assertEquals(sm.s16DecompressOneNumberWithHardCodes(output, 0, 22, 0), 28); + + assertEquals(sm.s16DecompressOneNumberWithHardCodesIntegrated(output, 0, 22, 0, 8, input), 28); + } } From d3e17b01022a6482dcebb3c78e6baa8c5b7a5f27 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 10:32:33 -0500 Subject: [PATCH 21/27] Add files via upload --- .../pfordelta/Simple16WithHardCodesTest.java | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java index 1635288..49abd1c 100644 --- a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java +++ b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java @@ -24,21 +24,31 @@ public void test(){ assertEquals(sm.s16Decompress(output, 0, input, 0, 1024), 28); - assertEquals(sm.s16DecompressWithIntBufferBackup(output, 0, 22, 1024), 28); - assertEquals(sm.s16DecompressWithIntBuffer(output, 0, 22, 1024), 28); - assertEquals(sm.s16DecompressWithIntBufferWithHardCodes(output, 0, 22, 1024), 28); - assertEquals(sm.s16DecompressWithIntBufferIntegrated(output, 0, 22, 1024, input, 8), 28); - assertEquals(sm.s16DecompressWithIntBufferIntegrated2(output, 0, 22, 1024, input, 8), 28); + for(int i = 0; i < 17; ++i){ - assertEquals(sm.s16DecompressWithIntBufferIntegratedBackup(output, 0, 22, 1024, input, 8), 28); + sm.s16DecompressWithIntBufferBackup(output, 0, i, 0); + + sm.s16DecompressWithIntBuffer(output, 0, i, 0); + + sm.s16DecompressWithIntBufferWithHardCodes(output, 0, i, 0); + + sm.s16DecompressWithIntBufferIntegrated(output, 0, i, 0, input, 8); + + + sm.s16DecompressWithIntBufferIntegrated2(output, 0, i, 0, input, 8); + + sm.s16DecompressWithIntBufferIntegratedBackup(output, 0, i, 0, input, 8); + + sm.s16DecompressOneNumberWithHardCodes(output, 0, i, 0); + + sm.s16DecompressOneNumberWithHardCodesIntegrated(output, 0, i, 0, 8, input); + } - assertEquals(sm.s16DecompressOneNumberWithHardCodes(output, 0, 22, 0), 28); - assertEquals(sm.s16DecompressOneNumberWithHardCodesIntegrated(output, 0, 22, 0, 8, input), 28); } } From 511a641e7f7aa17a270ad8fc5e97610d1d7d96a6 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 10:39:04 -0500 Subject: [PATCH 22/27] Update Simple16WithHardCodesTest.java --- .../java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java index 49abd1c..d6d38e8 100644 --- a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java +++ b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java @@ -29,6 +29,7 @@ public void test(){ for(int i = 0; i < 17; ++i){ + i = i << 28; sm.s16DecompressWithIntBufferBackup(output, 0, i, 0); From 912a98d39deb03b527b1de9236dca48df37728dc Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 10:43:37 -0500 Subject: [PATCH 23/27] Add files via upload --- .../com/kamikaze/pfordelta/Simple16WithHardCodesTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java index d6d38e8..2c3a381 100644 --- a/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java +++ b/src/test/java/com/kamikaze/pfordelta/Simple16WithHardCodesTest.java @@ -28,8 +28,9 @@ public void test(){ - for(int i = 0; i < 17; ++i){ - i = i << 28; + for(int j = 0; j < 17; j++){ + + int i = j << 28; sm.s16DecompressWithIntBufferBackup(output, 0, i, 0); From d13a5eb4ad285e152f58b672efb10a2298422c70 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 10:50:58 -0500 Subject: [PATCH 24/27] Add files via upload --- .../pfordelta/PForDeltaUnpack128Test.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128Test.java diff --git a/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128Test.java b/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128Test.java new file mode 100644 index 0000000..f0ad04a --- /dev/null +++ b/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128Test.java @@ -0,0 +1,32 @@ +package com.kamikaze.pfordelta; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Created by xueli on 12/8/16. + */ +public class PForDeltaUnpack128Test { + PForDeltaUnpack128 sm = new PForDeltaUnpack128(); + + @Test + public void test(){ + int[] input = new int[1024]; + int[] output = new int[1024]; + + for (int k = 0; k < 1024; ++k) input[k] = k; + + for(int i = 0; i < 29; i++){ + + //int i = j << 28; + + sm.unpack(output, input, i); + + + } + + + + } +} From ff7a9cbc2132d903822665efbc0e5e79d0350429 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 11:39:21 -0500 Subject: [PATCH 25/27] Add files via upload --- .../PForDeltaUnpack128WithIntBufferTest.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java diff --git a/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java b/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java new file mode 100644 index 0000000..d28c5ae --- /dev/null +++ b/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java @@ -0,0 +1,41 @@ +package com.kamikaze.pfordelta; + +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.IntBuffer; + +/** + * Created by xueli on 12/8/16. + */ +public class PForDeltaUnpack128WithIntBufferTest { + PForDeltaUnpack128WIthIntBuffer sm = new PForDeltaUnpack128WIthIntBuffer(); + + byte[] buf = new byte[16384]; + ByteBuffer bufWrap = ByteBuffer.wrap(buf); + + IntBuffer ibuf = bufWrap.asIntBuffer(); + + + @Test + public void test(){ + + //int[] input = new int[1024]; + int[] output = new int[256]; + + //for (int k = 0; k < 1024; ++k) input[k] = k; + + for(int i = 0; i < 29; i++){ + + //int i = j << 28; + + sm.unpack(output, ibuf, i); + + + } + + + + } +} From 87f2808b460b31779905a399d297f0454f88e523 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 11:42:14 -0500 Subject: [PATCH 26/27] Update PForDeltaUnpack128WithIntBufferTest.java --- .../kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java b/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java index d28c5ae..bc9fef3 100644 --- a/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java +++ b/src/test/java/com/kamikaze/pfordelta/PForDeltaUnpack128WithIntBufferTest.java @@ -12,7 +12,7 @@ public class PForDeltaUnpack128WithIntBufferTest { PForDeltaUnpack128WIthIntBuffer sm = new PForDeltaUnpack128WIthIntBuffer(); - byte[] buf = new byte[16384]; + byte[] buf = new byte[4096]; ByteBuffer bufWrap = ByteBuffer.wrap(buf); IntBuffer ibuf = bufWrap.asIntBuffer(); From 900d9d4fa41f41dfef8abcd48303ee60da94c108 Mon Sep 17 00:00:00 2001 From: YukiXueli Date: Fri, 9 Dec 2016 12:53:45 -0500 Subject: [PATCH 27/27] Add files via upload --- .../lemire/integercompression/DeltaTest.java | 93 ++++++++++ .../integercompression/GroupSimpleTest.java | 162 ++++++++++++++++++ .../IntegratedBitPackingTest.java | 152 ++++++++++++++++ .../SkippableBasicTest.java | 3 +- .../XorBinaryPackingTest.java | 5 - 5 files changed, 409 insertions(+), 6 deletions(-) create mode 100644 src/test/java/me/lemire/integercompression/DeltaTest.java create mode 100644 src/test/java/me/lemire/integercompression/GroupSimpleTest.java create mode 100644 src/test/java/me/lemire/integercompression/IntegratedBitPackingTest.java diff --git a/src/test/java/me/lemire/integercompression/DeltaTest.java b/src/test/java/me/lemire/integercompression/DeltaTest.java new file mode 100644 index 0000000..073fb54 --- /dev/null +++ b/src/test/java/me/lemire/integercompression/DeltaTest.java @@ -0,0 +1,93 @@ +package me.lemire.integercompression; + +import me.lemire.integercompression.differential.Delta; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + + +/** + * Tests for class Delta. + * + * @author Xue Li + */ +@SuppressWarnings({ "static-method" }) +public class DeltaTest { + + /** + * + */ + @Test + public void simpleDeltaTest() { + + int[] data = new int[1024]; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + Delta.delta(data); + + for (int k = 1; k < 1024; ++k){ + assertEquals("simple delta", data[k], 1); + } + } + + /** + * + */ + @Test + public void deltaWithInitialTest() { + int[] data = new int[1024]; //0 1 2 3 4 5 6 7 8 9 10 ... => 0 1 2 -5 1 1 1 1 8 9 10, return 7 + + int start = 3, length = 5, init; + int[] out = new int[length]; + + for (int k = 0; k < 1024; ++k) data[k] = k; + init = data[start - 1]; + + Delta.delta(data, start, length, init, out); + + for (int k = 0; k < length; ++k){ + assertEquals("deltaWithInitialTest", out[k], 1); + } + } + + @Test + public void inverseDeltaTest(){ + int[] data = new int[1024]; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + Delta.delta(data); + Delta.inverseDelta(data); + + for (int k = 0; k < 1024; ++k){ + assertEquals("inverseDeltaTest", data[k], k); + } + } + + @Test + public void fastinverseDeltaTest(){ + int[] data = new int[1024]; + + for (int k = 0; k < 1024; ++k) data[k] = k; + + Delta.delta(data); + Delta.fastinverseDelta(data); + + for (int k = 0; k < 1024; ++k){ + assertEquals("inverseDeltaTest", data[k], k); + } + } + + @Test + public void fastinverseDeltaWithInitialTest(){ + int[] data = {0, 1, 2, 1, 1, 1, 1, 1, 8, 9}; //0 1 2 3 4 5 6 7 8 9 10 ... => 0 1 2 -5 1 1 1 1 8 9 10, return 7 + int start = 3, length = 5, init = 2; + + Delta.fastinverseDelta(data, start, length, init); + + for (int k = 0; k < length; ++k){ + assertEquals("fastinverseDeltaWithInitialTest", data[k], k); + } + } +} diff --git a/src/test/java/me/lemire/integercompression/GroupSimpleTest.java b/src/test/java/me/lemire/integercompression/GroupSimpleTest.java new file mode 100644 index 0000000..91c864c --- /dev/null +++ b/src/test/java/me/lemire/integercompression/GroupSimpleTest.java @@ -0,0 +1,162 @@ +package me.lemire.integercompression; + +import org.junit.Test; + +import java.util.Arrays; + + +/** + * Tests for class GroupSimple9. + * + * @author Xue Li + */ +@SuppressWarnings({ "static-method" }) +public class GroupSimpleTest { + SkippableIntegerCODEC[] codecs = { + new GroupSimple9() + }; + + + /** + * + */ + @Test + public void consistentTest() { + int bitLength[] = { 1, 2, 3, 4, 5, 7, 9, 14, 28 }; + int codeNum[] = { 28, 14, 9, 7, 5, 4, 3, 2, 1 }; + for(int s1 = 0 ; s1 < 9; ++s1){ + for(int s2 = 0; s2 < 9; ++s2){ + int max1 = 1 << bitLength[s1]; + int max2 = 1 << bitLength[s2]; + int num1 = codeNum[s1]; + int num2 = codeNum[s2]; + + int N = num1 + num2; + if(N < 57) N = 57; + int[] data = new int[N]; + + int i = 0; + for(; i < num1; ++i){ + data[i] = max1 - 1; + } + for(; i < N; ++i){ + data[i] = max2 - 1; + } + + oneTest(N, data); + + } + } + + + } + private void oneTest(int N, int[] data){ + int[] rev = new int[N]; + + //for (int k = 0; k < N; ++k) + // data[k] = k % 128; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.consistentTest] codec = " + + c); + int[] outBuf = new int[N + 1024]; + for (int n = 0; n <= N; ++n) { + IntWrapper inPos = new IntWrapper(); + IntWrapper outPos = new IntWrapper(); + c.headlessCompress(data, inPos, n, outBuf, outPos); + + IntWrapper inPoso = new IntWrapper(); + IntWrapper outPoso = new IntWrapper(); + c.headlessUncompress(outBuf, inPoso, outPos.get(), rev, + outPoso, n); + if (outPoso.get() != n) { + throw new RuntimeException("bug "+n); + } + if (inPoso.get() != outPos.get()) { + throw new RuntimeException("bug "+n+" "+inPoso.get()+" "+outPos.get()); + } + for (int j = 0; j < n; ++j) + if (data[j] != rev[j]) { + throw new RuntimeException("bug"); + } + } + } + } + + /** + * + */ + @Test + public void varyingLengthTest() { + int N = 4096; + int[] data = new int[N]; + for (int k = 0; k < N; ++k) + data[k] = k; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.varyingLengthTest] codec = "+c); + for (int L = 1; L <= 128; L++) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug "+c.toString()+" "+k+" "+answer[k]+" "+data[k]); + } + for (int L = 128; L <= N; L *= 2) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug"); + } + + } + } + + /** + * + */ + /* + @Test + + public void varyingLengthTest2() { + int N = 128; + int[] data = new int[N]; + data[127] = -1; + for (SkippableIntegerCODEC c : codecs) { + System.out.println("[SkippeableBasicTest.varyingLengthTest2] codec = "+c); + + try { + // CODEC Simple9 is limited to "small" integers. + if (c.getClass().equals( + Class.forName("me.lemire.integercompression.Simple9"))) + continue; + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + try { + // CODEC Simple16 is limited to "small" integers. + if (c.getClass().equals( + Class.forName("me.lemire.integercompression.Simple16"))) + continue; + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + for (int L = 1; L <= 128; L++) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug at k = "+k+" "+answer[k]+" "+data[k]+" for "+c.toString()); + } + for (int L = 128; L <= N; L *= 2) { + int[] comp = TestUtils.compressHeadless(c, Arrays.copyOf(data, L)); + int[] answer = TestUtils.uncompressHeadless(c, comp, L); + for (int k = 0; k < L; ++k) + if (answer[k] != data[k]) + throw new RuntimeException("bug"); + } + + } + } + + */ +} diff --git a/src/test/java/me/lemire/integercompression/IntegratedBitPackingTest.java b/src/test/java/me/lemire/integercompression/IntegratedBitPackingTest.java new file mode 100644 index 0000000..d45e697 --- /dev/null +++ b/src/test/java/me/lemire/integercompression/IntegratedBitPackingTest.java @@ -0,0 +1,152 @@ +package me.lemire.integercompression; + +import me.lemire.integercompression.differential.Delta; +import me.lemire.integercompression.differential.IntegratedBitPacking; +import org.junit.Test; + +import java.text.DecimalFormat; +import java.util.Arrays; +import java.util.Random; + + +/** + * Tests for class GroupSimple9. + * + * @author Xue Li + */ +@SuppressWarnings({ "static-method" }) +public class IntegratedBitPackingTest { + + + /** + * + */ + @Test + public void Test() { + testCore(true); + } + private void testCore(boolean verbose) { + DecimalFormat dfspeed = new DecimalFormat("0"); + final int N = 32; + final int times = 100000; + Random r = new Random(0); + int[] data = new int[N]; + int[] compressed = new int[N]; + int[] uncompressed = new int[N]; + for (int bit = 0; bit < 32; ++bit) { + long comp = 0; + long compwm = 0; + long decomp = 0; + for (int t = 0; t < times; ++t) { + for (int k = 0; k < N; ++k) { + data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)); + } + long time1 = System.nanoTime(); + BitPacking + .fastpack(data, 0, compressed, 0, bit); + long time2 = System.nanoTime(); + BitPacking.fastpackwithoutmask(data, 0, + compressed, 0, bit); + long time3 = System.nanoTime(); + BitPacking.fastunpack(compressed, 0, + uncompressed, 0, bit); + long time4 = System.nanoTime(); + comp += time2 - time1; + compwm += time3 - time2; + decomp += time4 - time3; + } + if (verbose) + System.out.println("bit = " + + bit + + " comp. speed = " + + dfspeed.format(N * times * 1000.0 + / (comp)) + + " comp. speed wm = " + + dfspeed.format(N * times * 1000.0 + / (compwm)) + + " decomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (decomp))); + } + } + + /** + * + */ + @Test + public void varyingLengthTest() { + testWithDeltasCore(true); + } + + private void testWithDeltasCore(boolean verbose) { + DecimalFormat dfspeed = new DecimalFormat("0"); + final int N = 32; + final int times = 100000; + Random r = new Random(0); + int[] data = new int[N]; + int[] compressed = new int[N]; + int[] icompressed = new int[N]; + int[] uncompressed = new int[N]; + for (int bit = 1; bit < 32; ++bit) { + long comp = 0; + long decomp = 0; + long icomp = 0; + long idecomp = 0; + for (int t = 0; t < times; ++t) { + data[0] = r.nextInt(1 << (bit == 31 ? 30 : bit)); + for (int k = 1; k < N; ++k) { + data[k] = r.nextInt(1 << (bit == 31 ? 30 : bit)) + + data[k - 1]; + } + int[] tmpdata = Arrays + .copyOf(data, data.length); + long time1 = System.nanoTime(); + Delta.delta(tmpdata); + BitPacking.fastpackwithoutmask(tmpdata, 0, + compressed, 0, bit); + long time2 = System.nanoTime(); + BitPacking.fastunpack(compressed, 0, + uncompressed, 0, bit); + Delta.fastinverseDelta(uncompressed); + long time3 = System.nanoTime(); + if (!Arrays.equals(data, uncompressed)) + throw new RuntimeException("bug"); + comp += time2 - time1; + decomp += time3 - time2; + tmpdata = Arrays.copyOf(data, data.length); + time1 = System.nanoTime(); + if(bit == 26){ + int tttt = 0; + } + IntegratedBitPacking.integratedpack(0, tmpdata, + 0, icompressed, 0, bit); + time2 = System.nanoTime(); + IntegratedBitPacking.integratedunpack(0, + icompressed, 0, uncompressed, 0, bit); + time3 = System.nanoTime(); + if (!Arrays.equals(icompressed, compressed)) + throw new RuntimeException("ibug " + + bit); + if (!Arrays.equals(data, uncompressed)) + throw new RuntimeException("bug " + bit); + icomp += time2 - time1; + idecomp += time3 - time2; + } + if (verbose) + System.out.println("bit = " + + bit + + " comp. speed = " + + dfspeed.format(N * times * 1000.0 + / (comp)) + + " decomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (decomp)) + + " icomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (icomp)) + + " idecomp. speed = " + + dfspeed.format(N * times * 1000.0 + / (idecomp))); + } + } +} diff --git a/src/test/java/me/lemire/integercompression/SkippableBasicTest.java b/src/test/java/me/lemire/integercompression/SkippableBasicTest.java index d965992..d67a4e8 100644 --- a/src/test/java/me/lemire/integercompression/SkippableBasicTest.java +++ b/src/test/java/me/lemire/integercompression/SkippableBasicTest.java @@ -25,7 +25,8 @@ public class SkippableBasicTest { new SkippableComposition(new FastPFOR128(), new VariableByte()), new SkippableComposition(new FastPFOR(), new VariableByte()), new Simple9(), - new Simple16() }; + new Simple16() + }; /** diff --git a/src/test/java/me/lemire/integercompression/XorBinaryPackingTest.java b/src/test/java/me/lemire/integercompression/XorBinaryPackingTest.java index 3201b02..72f7871 100644 --- a/src/test/java/me/lemire/integercompression/XorBinaryPackingTest.java +++ b/src/test/java/me/lemire/integercompression/XorBinaryPackingTest.java @@ -20,11 +20,6 @@ public final class XorBinaryPackingTest { - - - - - private static void checkCompressAndUncompress(String label, int[] data) { XorBinaryPacking codec = new XorBinaryPacking(); int[] compBuf = TestUtils.compress(codec, data);