Skip to content

Commit e2d2d2f

Browse files
lemirelemire
committed
Adding more tests.
1 parent 196c41e commit e2d2d2f

6 files changed

Lines changed: 175 additions & 2 deletions

File tree

‎Makefile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ else
1717
CFLAGS += -O3
1818
endif
1919

20-
MAINEXECUTABLES=parse minify
20+
MAINEXECUTABLES=parse minify json2json
2121
TESTEXECUTABLES=jsoncheck numberparsingcheck stringparsingcheck
2222
COMPARISONEXECUTABLES=minifiercompetition parsingcompetition allparserscheckfile
2323

@@ -42,6 +42,7 @@ test: jsoncheck numberparsingcheck stringparsingcheck
4242
./numberparsingcheck
4343
./stringparsingcheck
4444
./jsoncheck
45+
./scripts/testjson2json.sh
4546
@echo
4647
@tput setaf 2
4748
@echo "It looks like the code is good!"
@@ -90,6 +91,10 @@ minifiercompetition: benchmark/minifiercompetition.cpp $(HEADERS) $(MINIFIERHEAD
9091
minify: tools/minify.cpp $(HEADERS) $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIBFILES)
9192
$(CXX) $(CXXFLAGS) -o minify $(MINIFIERLIBFILES) $(LIBFILES) tools/minify.cpp -I.
9293

94+
json2json: tools/json2json.cpp $(HEADERS) $(LIBFILES)
95+
$(CXX) $(CXXFLAGS) -o json2json $ tools/json2json.cpp $(LIBFILES) -I.
96+
97+
9398
ujdecode.o: $(UJSON4C_INCLUDE)
9499
$(CC) $(CFLAGS) -c dependencies/ujson4c/src/ujdecode.c
95100

‎include/simdjson/jsonformatutils.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ static inline void print_with_escapes(const unsigned char *src) {
99
putchar('\\');
1010
putchar('n');
1111
break;
12+
case '\r':
13+
putchar('\\');
14+
putchar('r');
15+
break;
1216
case '\"':
1317
putchar('\\');
1418
putchar('"');
@@ -23,7 +27,7 @@ static inline void print_with_escapes(const unsigned char *src) {
2327
break;
2428
default:
2529
if (*src <= 0x1F) {
26-
printf("\\u%x", *src);
30+
printf("\\u%04x", *src);
2731
} else
2832
putchar(*src);
2933
}

‎include/simdjson/parsedjson.h‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ struct ParsedJson {
123123
// print the json to stdout (should be valid)
124124
// return false if the tape is likely wrong (e.g., you did not parse a valid
125125
// JSON).
126+
WARN_UNUSED
126127
bool printjson() {
127128
size_t tapeidx = 0;
128129
u64 tape_val = tape[tapeidx];
@@ -224,6 +225,73 @@ struct ParsedJson {
224225
return true;
225226
}
226227

228+
bool dump_raw_tape() {
229+
size_t tapeidx = 0;
230+
u64 tape_val = tape[tapeidx++];
231+
u8 type = (tape_val >> 56);
232+
size_t howmany = 0;
233+
if (type == 'r') {
234+
howmany = tape_val & JSONVALUEMASK;
235+
} else {
236+
printf("Error: no starting root node?");
237+
return false;
238+
}
239+
for (; tapeidx < howmany; tapeidx++) {
240+
tape_val = tape[tapeidx];
241+
u64 payload = tape_val & JSONVALUEMASK;
242+
type = (tape_val >> 56);
243+
switch (type) {
244+
case '"': // we have a string
245+
printf("string: ");
246+
putchar('"');
247+
print_with_escapes((const unsigned char *)(string_buf + payload));
248+
putchar('"');
249+
printf("\n");
250+
break;
251+
case 'l': // we have a long int
252+
if (tapeidx + 1 >= howmany)
253+
return false;
254+
printf("integer: ");
255+
printf("%" PRId64, (int64_t)tape[++tapeidx]);
256+
break;
257+
case 'd': // we have a double
258+
printf("float: ");
259+
if (tapeidx + 1 >= howmany)
260+
return false;
261+
printf("%f", *((double *)&tape[++tapeidx]));
262+
break;
263+
case 'n': // we have a null
264+
printf("null");
265+
break;
266+
case 't': // we have a true
267+
printf("true");
268+
break;
269+
case 'f': // we have a false
270+
printf("false");
271+
break;
272+
case '{': // we have an object
273+
printf("{");
274+
break;
275+
case '}': // we end an object
276+
printf("}");
277+
break;
278+
case '[': // we start an array
279+
printf("[");
280+
break;
281+
case ']': // we end an array
282+
printf("]");
283+
break;
284+
case 'r': // we start and end with the root node
285+
printf("end of root");
286+
return false;
287+
default:
288+
return false;
289+
}
290+
}
291+
return true;
292+
}
293+
294+
227295
// all elements are stored on the tape using a 64-bit word.
228296
//
229297
// strings, double and ints are stored as

‎scripts/testjson2json.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
TMPDIR1=$(mktemp -d -t simdjson1)
4+
TMPDIR2=$(mktemp -d -t simdjson2)
5+
6+
trap "exit 1" HUP INT PIPE QUIT TERM
7+
trap "rm -rf $TMPDIR1 $TMPDIR2" EXIT
8+
9+
function founderror() {
10+
echo "code is wrong"
11+
exit 1
12+
}
13+
14+
make minify json2json
15+
for i in `cd jsonexamples && ls -1 *.json`; do
16+
echo $i
17+
./json2json jsonexamples/$i > $TMPDIR1/$i
18+
./json2json $TMPDIR1/$i > $TMPDIR2/$i
19+
cmp $TMPDIR1/$i $TMPDIR2/$i
20+
retVal=$?
21+
if [ $retVal -ne 0 ]; then
22+
founderror
23+
fi
24+
./minify $TMPDIR1/$i > $TMPDIR1/minify$i
25+
./minify $TMPDIR2/$i > $TMPDIR2/minify$i
26+
cmp $TMPDIR1/minify$i $TMPDIR2/minify$i
27+
retVal=$?
28+
if [ $retVal -ne 0 ]; then
29+
founderror
30+
fi
31+
./json2json $TMPDIR1/minify$i > $TMPDIR2/bisminify$i
32+
cmp $TMPDIR1/$i $TMPDIR2/bisminify$i
33+
retVal=$?
34+
if [ $retVal -ne 0 ]; then
35+
founderror
36+
fi
37+
done
38+
echo "test successful"
39+
40+
exit 0

‎src/stage34_unified.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,8 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
170170
}
171171
#ifdef SIMDJSON_ALLOWANYTHINGINROOT
172172
depth--; // for fall-through cases (e.g., documents containing just a string)
173+
pj.annotate_previousloc(pj.containing_scope_offset[depth],
174+
pj.get_current_loc());
173175
#endif // ALLOWANYTHINGINROOT
174176

175177
start_continue:

‎tools/json2json.cpp‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#include <unistd.h>
2+
3+
#include "simdjson/jsonparser.h"
4+
#include "simdjson/jsonioutil.h"
5+
6+
using namespace std;
7+
8+
int main(int argc, char *argv[]) {
9+
int c;
10+
bool rawdump = false;
11+
12+
while ((c = getopt (argc, argv, "d")) != -1)
13+
switch (c)
14+
{
15+
case 'd':
16+
rawdump = true;
17+
break;
18+
default:
19+
abort ();
20+
}
21+
if (optind >= argc) {
22+
cerr << "Reads json in, out the result of the parsing. " << endl;
23+
cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;
24+
exit(1);
25+
}
26+
const char * filename = argv[optind];
27+
if(optind + 1 < argc) {
28+
cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
29+
}
30+
std::string_view p;
31+
try {
32+
p = get_corpus(filename);
33+
} catch (const std::exception& e) { // caught by reference to base
34+
std::cout << "Could not load the file " << filename << std::endl;
35+
return EXIT_FAILURE;
36+
}
37+
ParsedJson pj;
38+
bool allocok = pj.allocateCapacity(p.size(), 1024);
39+
if(!allocok) {
40+
std::cerr << "failed to allocate memory" << std::endl;
41+
return EXIT_FAILURE;
42+
}
43+
bool is_ok = json_parse(p, pj); // do the parsing, return false on error
44+
if (!is_ok) {
45+
std::cerr << " Parsing failed. " << std::endl;
46+
return EXIT_FAILURE;
47+
}
48+
is_ok = rawdump ? pj.dump_raw_tape() : pj.printjson();
49+
if(!is_ok) {
50+
std::cerr << " Could not print out parsed result. " << std::endl;
51+
return EXIT_FAILURE;
52+
}
53+
return EXIT_SUCCESS;
54+
}

0 commit comments

Comments
 (0)