Skip to content

Commit f641070

Browse files
jkeiserjkeiser
committed
Create basic streaming parser
1 parent 6797a6a commit f641070

33 files changed

Lines changed: 1627 additions & 321 deletions

‎include/simdjson.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
2424
#include "simdjson/dom/element.h"
2525
#include "simdjson/dom/object.h"
2626
#include "simdjson/dom/parser.h"
27+
#include "simdjson/stream/array.h"
28+
#include "simdjson/stream/document.h"
29+
#include "simdjson/stream/documents.h"
30+
#include "simdjson/stream/element.h"
31+
#include "simdjson/stream/field.h"
32+
#include "simdjson/stream/object.h"
33+
#include "simdjson/stream/raw_json_string.h"
2734

2835
// Deprecated API
2936
#include "simdjson/dom/jsonparser.h"
@@ -41,6 +48,11 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
4148
#include "simdjson/inline/parsedjson_iterator.h"
4249
#include "simdjson/inline/parser.h"
4350
#include "simdjson/inline/tape_ref.h"
51+
#include "simdjson/stream/array-inl.h"
52+
#include "simdjson/stream/document-inl.h"
53+
#include "simdjson/stream/documents-inl.h"
54+
#include "simdjson/stream/element-inl.h"
55+
#include "simdjson/stream/object-inl.h"
4456

4557
SIMDJSON_POP_DISABLE_WARNINGS
4658

‎include/simdjson/dom/parser.h‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
#include <string>
1515

1616
namespace simdjson {
17+
namespace stream {
18+
class document;
19+
} // namespace stream
1720

1821
namespace dom {
1922

@@ -91,6 +94,7 @@ class parser {
9194
*/
9295
inline simdjson_result<element> load(const std::string &path) & noexcept;
9396
inline simdjson_result<element> load(const std::string &path) && = delete ;
97+
9498
/**
9599
* Parse a JSON document and return a temporary reference to it.
96100
*
@@ -141,6 +145,58 @@ class parser {
141145
/** @private We do not want to allow implicit conversion from C string to std::string. */
142146
really_inline simdjson_result<element> parse(const char *buf) noexcept = delete;
143147

148+
/**
149+
* Parse a JSON document and return a temporary reference to it.
150+
*
151+
* dom::parser parser;
152+
* for (element doc : parser.stream_many(buf, len)) {
153+
* ...
154+
* }
155+
*
156+
* ### IMPORTANT: Document Lifetime
157+
*
158+
* The JSON document still lives in the parser: this is the most efficient way to parse JSON
159+
* documents because it reuses the same buffers, but you *must* use the document before you
160+
* destroy the parser or call stream_many() again.
161+
*
162+
* ### REQUIRED: Buffer Padding
163+
*
164+
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
165+
* those bytes are initialized to, as long as they are allocated.
166+
*
167+
* If realloc_if_needed is true, it is assumed that the buffer does *not* have enough padding,
168+
* and it is copied into an enlarged temporary buffer before parsing.
169+
*
170+
* ### Parser Capacity
171+
*
172+
* If the parser's current capacity is less than len, it will allocate enough capacity
173+
* to handle it (up to max_capacity).
174+
*
175+
* @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
176+
* realloc_if_needed is true.
177+
* @param len The length of the JSON.
178+
* @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
179+
* @return The document, or an error:
180+
* - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity,
181+
* and memory allocation fails.
182+
* - CAPACITY if the parser does not have enough capacity and len > max_capacity.
183+
* - other json errors if parsing fails.
184+
*/
185+
inline simdjson_result<stream::document> stream(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept;
186+
inline simdjson_result<stream::document> stream(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete;
187+
/** @overload stream_many(const uint8_t *buf, size_t len, bool realloc_if_needed) */
188+
really_inline simdjson_result<stream::document> stream(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept;
189+
really_inline simdjson_result<stream::document> stream(const char *buf, size_t len, bool realloc_if_needed = true) && =delete;
190+
/** @overload stream_many(const uint8_t *buf, size_t len, bool realloc_if_needed) */
191+
really_inline simdjson_result<stream::document> stream(const std::string &s) & noexcept;
192+
really_inline simdjson_result<stream::document> stream(const std::string &s) && =delete;
193+
/** @overload stream_many(const uint8_t *buf, size_t len, bool realloc_if_needed) */
194+
really_inline simdjson_result<stream::document> stream(const padded_string &s) & noexcept;
195+
really_inline simdjson_result<stream::document> stream(const padded_string &s) && =delete;
196+
197+
/** @private We do not want to allow implicit conversion from C string to std::string. */
198+
really_inline simdjson_result<stream::document> stream(const char *buf) noexcept = delete;
199+
144200
/**
145201
* Load a file containing many JSON documents.
146202
*

‎include/simdjson/inline/parser.h‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "simdjson/implementation.h"
77
#include "simdjson/internal/jsonformatutils.h"
88
#include "simdjson/portability.h"
9+
#include "simdjson/stream/document.h"
910
#include <cstdio>
1011
#include <climits>
1112

@@ -141,6 +142,35 @@ inline simdjson_result<document_stream> parser::parse_many(const padded_string &
141142
return parse_many(s.data(), s.length(), batch_size);
142143
}
143144

145+
inline simdjson_result<stream::document> parser::stream(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept {
146+
error_code code = ensure_capacity(len);
147+
if (code) { return { stream::document(*this, buf), code }; }
148+
149+
if (realloc_if_needed) {
150+
const uint8_t *tmp_buf = buf;
151+
buf = (uint8_t *)internal::allocate_padded_buffer(len);
152+
if (buf == nullptr) {
153+
return { stream::document(*this, buf), MEMALLOC };
154+
}
155+
memcpy((void *)buf, tmp_buf, len);
156+
}
157+
158+
code = implementation->stage1(buf, len, false);
159+
if (realloc_if_needed) {
160+
aligned_free((void *)buf); // must free before we exit
161+
}
162+
return { stream::document(*this, buf), code };
163+
}
164+
really_inline simdjson_result<stream::document> parser::stream(const char *buf, size_t len, bool realloc_if_needed) & noexcept {
165+
return stream((const uint8_t *)buf, len, realloc_if_needed);
166+
}
167+
really_inline simdjson_result<stream::document> parser::stream(const std::string &s) & noexcept {
168+
return stream(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
169+
}
170+
really_inline simdjson_result<stream::document> parser::stream(const padded_string &s) & noexcept {
171+
return stream(s.data(), s.length(), false);
172+
}
173+
144174
really_inline size_t parser::capacity() const noexcept {
145175
return implementation ? implementation->capacity() : 0;
146176
}

‎include/simdjson/internal/json_iterator.h‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#ifndef SIMDJSON_INTERNAL_JSON_ITERATOR_H
2+
#define SIMDJSON_INTERNAL_JSON_ITERATOR_H
3+
4+
#include "simdjson/common_defs.h"
5+
6+
namespace simdjson {
7+
namespace internal {
8+
9+
class json_iterator {
10+
public:
11+
really_inline json_iterator(const uint32_t *_structural_index, const uint8_t *_buf, uint8_t *_string_buf)
12+
: structural_index{_structural_index}, buf{_buf}, string_buf{_string_buf} {}
13+
json_iterator() = delete;
14+
really_inline const uint8_t * advance() noexcept { printf("advance(%c)\n", buf[*structural_index]); return &buf[*(structural_index++)]; }
15+
really_inline const uint8_t * get() const noexcept { printf("get(%c)\n", buf[*structural_index]); return &buf[*structural_index]; }
16+
really_inline const uint8_t * peek_prev() const noexcept { return &buf[*(structural_index-1)]; }
17+
really_inline const uint8_t * peek_next() const noexcept { return &buf[*(structural_index+1)]; }
18+
19+
// For iterator interface
20+
really_inline bool operator!=(json_iterator &other) const noexcept { return structural_index != other.structural_index; }
21+
22+
/** The current structural indexes */
23+
const uint32_t * structural_index;
24+
/** The buffer the structural indexes point at */
25+
const uint8_t * const buf;
26+
/**
27+
* The current location in the buffer to write strings to.
28+
*
29+
* This is why it cannot be copied: if there are multiple copies of this, then strings could
30+
* potentially copy over each other
31+
*/
32+
uint8_t *string_buf;
33+
}; // class structurals
34+
35+
} // namespace internal
36+
} // namespace simdjson
37+
38+
#endif // SIMDJSON_INTERNAL_JSON_ITERATOR_H

‎include/simdjson/internal/jsoncharutils.h‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#ifndef SIMDJSON_INTERNAL_JSONCHARUTILS_H
2+
#define SIMDJSON_INTERNAL_JSONCHARUTILS_H
3+
4+
#include "simdjson/common_defs.h"
5+
#include "simdjson/portability.h"
6+
7+
namespace simdjson {
8+
namespace internal {
9+
10+
// structural chars here are
11+
// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL)
12+
// we are also interested in the four whitespace characters
13+
// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
14+
15+
constexpr const uint32_t structural_or_whitespace[256] = {
16+
0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
17+
0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
18+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
19+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,
20+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
21+
0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
22+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
23+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
24+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
25+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
26+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
27+
28+
really_inline uint32_t is_structural_or_whitespace(uint8_t c) {
29+
return structural_or_whitespace[c];
30+
}
31+
32+
constexpr const uint32_t structural_or_whitespace_negated[256] = {
33+
1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,
34+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
35+
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
36+
37+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
38+
1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
39+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1,
40+
41+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
42+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
43+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
44+
45+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
46+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
47+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
48+
49+
// return non-zero if not a structural or whitespace char
50+
// zero otherwise
51+
really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) {
52+
return structural_or_whitespace_negated[c];
53+
}
54+
55+
} // namespace internal
56+
} // namespace simdjson
57+
58+
#endif // SIMDJSON_INTERNAL_JSONCHARUTILS_H
59+

0 commit comments

Comments
 (0)