Skip to content

Commit 69a247d

Browse files
author
Daniel Lemire
committed
Adding tests.
1 parent a76c67c commit 69a247d

2 files changed

Lines changed: 266 additions & 0 deletions

File tree

‎tests/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ add_cpp_test(jsoncheck LABELS acceptance per_implementation)
5858
add_cpp_test(parse_many_test LABELS acceptance per_implementation)
5959
add_cpp_test(pointercheck LABELS acceptance per_implementation)
6060
add_cpp_test(extracting_values_example LABELS acceptance per_implementation)
61+
add_cpp_test(unicode_tests LABELS acceptance per_implementation)
62+
6163
find_program(BASH bash)
6264

6365

‎tests/unicode_tests.cpp‎

Lines changed: 264 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,264 @@
1+
#include "simdjson.h"
2+
#include <cstddef>
3+
#include <cstdint>
4+
#include <random>
5+
6+
class RandomUTF8 final {
7+
public:
8+
RandomUTF8(std::random_device &rd, int prob_1byte, int prob_2bytes,
9+
int prob_3bytes, int prob_4bytes);
10+
11+
std::vector<uint8_t> generate(size_t output_bytes);
12+
std::vector<uint8_t> generate(size_t output_bytes, long seed);
13+
14+
private:
15+
uint32_t generate();
16+
17+
std::mt19937 gen;
18+
std::discrete_distribution<> bytes_count;
19+
std::uniform_int_distribution<uint8_t> val_7bit{0x00, 0x7f}; // 0b0xxxxxxx
20+
std::uniform_int_distribution<uint8_t> val_6bit{0x00, 0x3f}; // 0b10xxxxxx
21+
std::uniform_int_distribution<uint8_t> val_5bit{0x00, 0x1f}; // 0b110xxxxx
22+
std::uniform_int_distribution<uint8_t> val_4bit{0x00, 0x0f}; // 0b1110xxxx
23+
std::uniform_int_distribution<uint8_t> val_3bit{0x00, 0x07}; // 0b11110xxx
24+
};
25+
26+
RandomUTF8::RandomUTF8(std::random_device &rd, int prob_1byte, int prob_2bytes,
27+
int prob_3bytes, int prob_4bytes)
28+
: gen(rd()), bytes_count({double(prob_1byte), double(prob_2bytes),
29+
double(prob_3bytes), double(prob_4bytes)}) {}
30+
31+
std::vector<uint8_t> RandomUTF8::generate(size_t output_bytes) {
32+
std::vector<uint8_t> result;
33+
result.reserve(output_bytes);
34+
uint8_t candidate, head;
35+
while (result.size() < output_bytes) {
36+
switch (bytes_count(gen)) {
37+
case 0: // 1 byte
38+
candidate = val_7bit(gen);
39+
while (candidate == 0) { // though strictly speaking, a stream of nulls is
40+
// UTF8, it tends to break some code
41+
candidate = val_7bit(gen);
42+
}
43+
result.push_back(candidate);
44+
break;
45+
case 1: // 2 bytes
46+
candidate = 0xc0 | val_5bit(gen);
47+
while (candidate < 0xC2) {
48+
candidate = 0xc0 | val_5bit(gen);
49+
}
50+
result.push_back(candidate);
51+
result.push_back(0x80 | val_6bit(gen));
52+
break;
53+
case 2: // 3 bytes
54+
head = 0xe0 | val_4bit(gen);
55+
result.push_back(head);
56+
candidate = 0x80 | val_6bit(gen);
57+
if (head == 0xE0) {
58+
while (candidate < 0xA0) {
59+
candidate = 0x80 | val_6bit(gen);
60+
}
61+
} else if (head == 0xED) {
62+
while (candidate > 0x9F) {
63+
candidate = 0x80 | val_6bit(gen);
64+
}
65+
}
66+
result.push_back(candidate);
67+
result.push_back(0x80 | val_6bit(gen));
68+
break;
69+
case 3: // 4 bytes
70+
head = 0xf0 | val_3bit(gen);
71+
while (head > 0xF4) {
72+
head = 0xf0 | val_3bit(gen);
73+
}
74+
result.push_back(head);
75+
candidate = 0x80 | val_6bit(gen);
76+
if (head == 0xF0) {
77+
while (candidate < 0x90) {
78+
candidate = 0x80 | val_6bit(gen);
79+
}
80+
} else if (head == 0xF4) {
81+
while (candidate > 0x8F) {
82+
candidate = 0x80 | val_6bit(gen);
83+
}
84+
}
85+
result.push_back(candidate);
86+
result.push_back(0x80 | val_6bit(gen));
87+
result.push_back(0x80 | val_6bit(gen));
88+
break;
89+
}
90+
}
91+
result.push_back(0); // EOS for scalar code
92+
93+
return result;
94+
}
95+
96+
std::vector<uint8_t> RandomUTF8::generate(size_t output_bytes, long seed) {
97+
gen.seed(uint32_t(seed));
98+
return generate(output_bytes);
99+
}
100+
101+
WARN_UNUSED bool basic_validate_utf8(const char *buf, size_t len) noexcept {
102+
const uint8_t *data = (const uint8_t *)buf;
103+
uint64_t pos = 0;
104+
uint64_t next_pos = 0;
105+
uint32_t code_point = 0;
106+
while (pos < len) {
107+
unsigned char byte = data[pos];
108+
if (byte < 0b10000000) {
109+
pos++;
110+
continue;
111+
} else if ((byte & 0b11100000) == 0b11000000) {
112+
next_pos = pos + 2;
113+
if (next_pos > len) {
114+
return false;
115+
}
116+
if ((data[pos + 1] & 0b11000000) != 0b10000000) {
117+
return false;
118+
}
119+
// range check
120+
code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
121+
if (code_point < 0x80 || 0x7ff < code_point) {
122+
return false;
123+
}
124+
} else if ((byte & 0b11110000) == 0b11100000) {
125+
next_pos = pos + 3;
126+
if (next_pos > len) {
127+
return false;
128+
}
129+
if ((data[pos + 1] & 0b11000000) != 0b10000000) {
130+
return false;
131+
}
132+
if ((data[pos + 2] & 0b11000000) != 0b10000000) {
133+
return false;
134+
}
135+
// range check
136+
code_point = (byte & 0b00001111) << 12 |
137+
(data[pos + 1] & 0b00111111) << 6 |
138+
(data[pos + 2] & 0b00111111);
139+
if (code_point < 0x800 || 0xffff < code_point ||
140+
(0xd7ff < code_point && code_point < 0xe000)) {
141+
return false;
142+
}
143+
} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
144+
next_pos = pos + 4;
145+
if (next_pos > len) {
146+
return false;
147+
}
148+
if ((data[pos + 1] & 0b11000000) != 0b10000000) {
149+
return false;
150+
}
151+
if ((data[pos + 2] & 0b11000000) != 0b10000000) {
152+
return false;
153+
}
154+
if ((data[pos + 3] & 0b11000000) != 0b10000000) {
155+
return false;
156+
}
157+
// range check
158+
code_point =
159+
(byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
160+
(data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
161+
if (code_point < 0xffff || 0x10ffff < code_point) {
162+
return false;
163+
}
164+
} else {
165+
// we may have a continuation
166+
return false;
167+
}
168+
pos = next_pos;
169+
}
170+
return true;
171+
}
172+
173+
void brute_force_tests() {
174+
printf("running brute-force UTF-8 tests... ");
175+
fflush(NULL);
176+
std::random_device rd{};
177+
RandomUTF8 gen_1_2_3_4(rd, 1, 1, 1, 1);
178+
size_t total = 1000;
179+
for (size_t i = 0; i < total; i++) {
180+
181+
auto UTF8 = gen_1_2_3_4.generate(rand() % 256);
182+
if (!simdjson::validate_utf8((const char *)UTF8.data(), UTF8.size())) {
183+
std::cerr << "bug" << std::endl;
184+
abort();
185+
}
186+
for (size_t flip = 0; flip < 1000; ++flip) {
187+
// we are going to hack the string as long as it is UTF-8
188+
UTF8[rand() % UTF8.size()] ^= uint8_t(1)
189+
<< (rand() % 8); // we flip exactly one bit
190+
bool is_ok =
191+
simdjson::validate_utf8((const char *)UTF8.data(), UTF8.size());
192+
bool is_ok_basic =
193+
basic_validate_utf8((const char *)UTF8.data(), UTF8.size());
194+
if (is_ok != is_ok_basic) {
195+
std::cerr << "bug" << std::endl;
196+
abort();
197+
}
198+
}
199+
}
200+
printf("tests ok.\n");
201+
}
202+
203+
void test() {
204+
printf("running hard-coded UTF-8 tests... ");
205+
fflush(NULL);
206+
// additional tests are from autobahn websocket testsuite
207+
// https://github.com/crossbario/autobahn-testsuite/tree/master/autobahntestsuite/autobahntestsuite/case
208+
const char *goodsequences[] = {"a",
209+
"\xc3\xb1",
210+
"\xe2\x82\xa1",
211+
"\xf0\x90\x8c\xbc",
212+
"안녕하세요, 세상",
213+
"\xc2\x80", // 6.7.2
214+
"\xf0\x90\x80\x80", // 6.7.4
215+
"\xee\x80\x80", // 6.11.2
216+
"\xef\xbb\xbf"};
217+
const char *badsequences[] = {
218+
"\xc3\x28", // 0
219+
"\xa0\xa1", // 1
220+
"\xe2\x28\xa1", // 2
221+
"\xe2\x82\x28", // 3
222+
"\xf0\x28\x8c\xbc", // 4
223+
"\xf0\x90\x28\xbc", // 5
224+
"\xf0\x28\x8c\x28", // 6
225+
"\xc0\x9f", // 7
226+
"\xf5\xff\xff\xff", // 8
227+
"\xed\xa0\x81", // 9
228+
"\xf8\x90\x80\x80\x80", // 10
229+
"123456789012345\xed", // 11
230+
"123456789012345\xf1", // 12
231+
"123456789012345\xc2", // 13
232+
"\xC2\x7F", // 14
233+
"\xce", // 6.6.1
234+
"\xce\xba\xe1", // 6.6.3
235+
"\xce\xba\xe1\xbd", // 6.6.4
236+
"\xce\xba\xe1\xbd\xb9\xcf", // 6.6.6
237+
"\xce\xba\xe1\xbd\xb9\xcf\x83\xce", // 6.6.8
238+
"\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce", // 6.6.10
239+
"\xdf", // 6.14.6
240+
"\xef\xbf", // 6.14.7
241+
"\x80",
242+
"\x91\x85\x95\x9e",
243+
"\x6c\x02\x8e\x18"};
244+
for (size_t i = 0; i < 9; i++) {
245+
size_t len = strlen(goodsequences[i]);
246+
if (!simdjson::validate_utf8(goodsequences[i], len)) {
247+
printf("bug goodsequences[%zu]\n", i);
248+
abort();
249+
}
250+
}
251+
for (size_t i = 0; i < 26; i++) {
252+
size_t len = strlen(badsequences[i]);
253+
if (simdjson::validate_utf8(badsequences[i], len)) {
254+
printf("bug lookup2 badsequences[%zu]\n", i);
255+
abort();
256+
}
257+
}
258+
printf("tests ok.\n");
259+
}
260+
int main() {
261+
brute_force_tests();
262+
test();
263+
return EXIT_SUCCESS;
264+
}

0 commit comments

Comments
 (0)