Skip to content

Commit 3a2f746

Browse files
lemirelemire
committed
Trying with just a unified stage 3.
1 parent c5a396e commit 3a2f746

2 files changed

Lines changed: 71 additions & 6 deletions

File tree

‎src/jsonparser.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,12 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne
4040
}
4141
}
4242
bool isok = find_structural_bits(buf, len, pj);
43-
if (isok) {
43+
/*if (isok) {
4444
isok = flatten_indexes(len, pj);
4545
} else {
4646
if(reallocated) free((void*)buf);
4747
return false;
48-
}
48+
}*/
4949
if (isok) {
5050
isok = unified_machine(buf, len, pj);
5151
} else {

‎src/stage1_find_marks.cpp‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,35 @@
77
#define SIMDJSON_UTF8VALIDATE
88
#endif
99

10+
#ifndef NO_PDEP_WIDTH
11+
#define NO_PDEP_WIDTH 8
12+
#endif
13+
14+
#define SET_BIT(i) \
15+
base_ptr[base + i] = (uint32_t)idx + trailingzeroes(structurals); \
16+
structurals = structurals & (structurals - 1);
17+
18+
#define SET_BIT1 SET_BIT(0)
19+
#define SET_BIT2 SET_BIT1 SET_BIT(1)
20+
#define SET_BIT3 SET_BIT2 SET_BIT(2)
21+
#define SET_BIT4 SET_BIT3 SET_BIT(3)
22+
#define SET_BIT5 SET_BIT4 SET_BIT(4)
23+
#define SET_BIT6 SET_BIT5 SET_BIT(5)
24+
#define SET_BIT7 SET_BIT6 SET_BIT(6)
25+
#define SET_BIT8 SET_BIT7 SET_BIT(7)
26+
#define SET_BIT9 SET_BIT8 SET_BIT(8)
27+
#define SET_BIT10 SET_BIT9 SET_BIT(9)
28+
#define SET_BIT11 SET_BIT10 SET_BIT(10)
29+
#define SET_BIT12 SET_BIT11 SET_BIT(11)
30+
#define SET_BIT13 SET_BIT12 SET_BIT(12)
31+
#define SET_BIT14 SET_BIT13 SET_BIT(13)
32+
#define SET_BIT15 SET_BIT14 SET_BIT(14)
33+
#define SET_BIT16 SET_BIT15 SET_BIT(15)
34+
35+
#define CALL(macro, ...) macro(__VA_ARGS__)
36+
37+
#define SET_BITLOOPN(n) SET_BIT##n
38+
1039
// It seems that many parsers do UTF-8 validation.
1140
// RapidJSON does not do it by default, but a flag
1241
// allows it.
@@ -33,11 +62,13 @@ WARN_UNUSED
3362
cerr << "Your ParsedJson object only supports documents up to "<< pj.bytecapacity << " bytes but you are trying to process " << len << " bytes\n";
3463
return false;
3564
}
65+
uint32_t *base_ptr = pj.structural_indexes;
66+
uint32_t base = 0;
3667
#ifdef SIMDJSON_UTF8VALIDATE
3768
__m256i has_error = _mm256_setzero_si256();
3869
struct avx_processed_utf_bytes previous;
39-
previous.rawbytes = _mm256_setzero_si256();
40-
previous.high_nibbles = _mm256_setzero_si256();
70+
previous.rawbytes = _mm256_setzero_si256();
71+
previous.high_nibbles = _mm256_setzero_si256();
4172
previous.carried_continuations = _mm256_setzero_si256();
4273
#endif
4374

@@ -201,7 +232,19 @@ WARN_UNUSED
201232
// now, we've used our close quotes all we need to. So let's switch them off
202233
// they will be off in the quote mask and on in quote bits.
203234
structurals &= ~(quote_bits & ~quote_mask);
204-
*(uint64_t *)(pj.structurals + idx / 8) = structurals;
235+
236+
uint32_t cnt = hamming(structurals);
237+
uint32_t next_base = base + cnt;
238+
while (structurals) {
239+
CALL(SET_BITLOOPN, NO_PDEP_WIDTH)
240+
/*for(size_t i = 0; i < NO_PDEP_WIDTH; i++) {
241+
base_ptr[base+i] = (uint32_t)idx + trailingzeroes(s);
242+
s = s & (s - 1);
243+
}*/
244+
base += NO_PDEP_WIDTH;
245+
}
246+
base = next_base;
247+
//*(uint64_t *)(pj.structurals + idx / 8) = structurals;
205248
}
206249

207250
////////////////
@@ -354,8 +397,30 @@ WARN_UNUSED
354397
// now, we've used our close quotes all we need to. So let's switch them off
355398
// they will be off in the quote mask and on in quote bits.
356399
structurals &= ~(quote_bits & ~quote_mask);
357-
*(uint64_t *)(pj.structurals + idx / 8) = structurals;
400+
//*(uint64_t *)(pj.structurals + idx / 8) = structurals;
401+
uint32_t cnt = hamming(structurals);
402+
uint32_t next_base = base + cnt;
403+
while (structurals) {
404+
CALL(SET_BITLOOPN, NO_PDEP_WIDTH)
405+
/*for(size_t i = 0; i < NO_PDEP_WIDTH; i++) {
406+
base_ptr[base+i] = (uint32_t)idx + trailingzeroes(s);
407+
s = s & (s - 1);
408+
}*/
409+
base += NO_PDEP_WIDTH;
410+
}
411+
base = next_base;
412+
}
413+
pj.n_structural_indexes = base;
414+
if(base_ptr[pj.n_structural_indexes-1] > len) {
415+
fprintf( stderr,"Internal bug\n");
416+
return false;
358417
}
418+
if(len != base_ptr[pj.n_structural_indexes-1]) {
419+
// the string might not be NULL terminated, but we add a virtual NULL ending character.
420+
base_ptr[pj.n_structural_indexes++] = len;
421+
}
422+
base_ptr[pj.n_structural_indexes] = 0; // make it safe to dereference one beyond this array
423+
359424
#ifdef SIMDJSON_UTF8VALIDATE
360425
return _mm256_testz_si256(has_error, has_error);
361426
#else

0 commit comments

Comments
 (0)