Skip to content

Commit 29fc515

Browse files
piotte13piotte13
authored
Introducing concurrency mode in JsonStream. (simdjson#373)
* JsonStream threaded prototype * JsonStream Threaded version working. Still supporting non-threaded version. * Fix where invalid files would enter infinite loop. * SingleHeader update * I will remove -pthread in cmake for now. * Attempt at resolving the -pthread issue
1 parent 6cd8fb7 commit 29fc515

14 files changed

Lines changed: 1080 additions & 608 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
.vscode/
2+
.idea/
23
/VS/
34
/build/
45
/benchbranch/

‎CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ option(SIMDJSON_DISABLE_AVX "Forcefully disable AVX even if hardware supports it
1111
set(CMAKE_CXX_STANDARD 17)
1212
set(CMAKE_CXX_STANDARD_REQUIRED ON)
1313
set(CMAKE_MACOSX_RPATH OFF)
14+
set(CMAKE_THREAD_PREFER_PTHREAD ON)
15+
set(THREADS_PREFER_PTHREAD_FLAG ON)
16+
1417
if (NOT CMAKE_BUILD_TYPE)
1518
message(STATUS "No build type selected, default to Release")
1619
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
@@ -35,6 +38,11 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake")
3538

3639
find_package(CTargets)
3740
find_package(Options)
41+
find_package(Threads REQUIRED)
42+
43+
if(CMAKE_USE_PTHREADS_INIT)
44+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
45+
endif()
3846

3947
install(DIRECTORY include/${SIMDJSON_LIB_NAME} DESTINATION include)
4048
set (TEST_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/")

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ else
2222
ARCHFLAGS ?= -msse4.2 -mpclmul # lowest supported feature set?
2323
endif
2424

25-
CXXFLAGS = $(ARCHFLAGS) -std=c++17 -Wall -Wextra -Wshadow -Iinclude -Isrc -Ibenchmark/linux $(EXTRAFLAGS)
25+
CXXFLAGS = $(ARCHFLAGS) -std=c++17 -pthread -Wall -Wextra -Wshadow -Iinclude -Isrc -Ibenchmark/linux $(EXTRAFLAGS)
2626
CFLAGS = $(ARCHFLAGS) -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS)
2727

2828

‎amalgamation.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -181,16 +181,16 @@ echo "Giving final instructions:"
181181
CPPBIN=${DEMOCPP%%.*}
182182

183183
echo "Try :"
184-
echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson"
184+
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson"
185185

186186
SINGLEHDR=$SCRIPTPATH/singleheader
187187
echo "Copying files to $SCRIPTPATH/singleheader "
188188
mkdir -p $SINGLEHDR
189-
echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md
189+
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md
190190
cp ${AMAL_C} ${AMAL_H} ${DEMOCPP} $SINGLEHDR
191191
ls $SINGLEHDR
192192

193-
cd $SINGLEHDR && c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
193+
cd $SINGLEHDR && c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
194194

195195
lowercase(){
196196
echo "$1" | tr 'A-Z' 'a-z'

‎benchmark/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,7 @@ target_include_directories(${SIMDJSON_LIB_NAME}
77
add_cpp_benchmark(parse)
88
add_cpp_benchmark(statisticalmodel)
99
add_cpp_benchmark(parse_stream)
10+
11+
target_link_libraries(parse_stream Threads::Threads)
12+
1013
add_executable(perfdiff perfdiff.cpp)

‎benchmark/parse_stream.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
#include "simdjson/parsedjson.h"
99

1010
#define NB_ITERATION 5
11-
#define MIN_BATCH_SIZE 100000
11+
#define MIN_BATCH_SIZE 200000
1212
#define MAX_BATCH_SIZE 10000000
1313

1414
bool test_baseline = false;
@@ -74,7 +74,7 @@ if(test_per_batch) {
7474
std::wclog << "Jsonstream: Speed per batch_size... from " << MIN_BATCH_SIZE
7575
<< " bytes to " << MAX_BATCH_SIZE << " bytes..." << std::endl;
7676
std::cout << "Batch Size\t" << "Gigabytes/second\t" << "Nb of documents parsed" << std::endl;
77-
for (size_t i = MIN_BATCH_SIZE; i <= MAX_BATCH_SIZE; i += (MAX_BATCH_SIZE - MIN_BATCH_SIZE) / 200) {
77+
for (size_t i = MIN_BATCH_SIZE; i <= MAX_BATCH_SIZE; i += (MAX_BATCH_SIZE - MIN_BATCH_SIZE) / 30) {
7878
batch_size_res.insert(std::pair<size_t, double>(i, 0));
7979
int count;
8080
for (size_t j = 0; j < 5; j++) {

‎include/simdjson/jsonstream.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44

55
#include <algorithm>
6+
#include <thread>
67
#include "simdjson/stage1_find_marks.h"
78
#include "simdjson/stage2_build_tape.h"
89
#include "simdjson/simdjson.h"
@@ -35,8 +36,17 @@ namespace simdjson {
3536
bool error_on_last_attempt{false};
3637
bool load_next_batch{true};
3738
size_t current_buffer_loc{0};
39+
size_t last_json_buffer_loc{0};
40+
size_t thread_current_buffer_loc{0};
3841
size_t n_parsed_docs{0};
3942
size_t n_bytes_parsed{0};
43+
44+
std::thread stage_1_thread;
45+
simdjson::ParsedJson pj_thread;
46+
47+
#ifdef SIMDJSON_THREADS_ENABLED
48+
size_t find_last_json(const ParsedJson &pj);
49+
#endif
4050
};
4151

4252
}

‎include/simdjson/portability.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@
4242
#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul")
4343
#define TARGET_ARM64
4444

45+
// Is threading enabled?
46+
#if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT)
47+
#define SIMDJSON_THREADS_ENABLED 1
48+
#endif
49+
4550
#ifdef _MSC_VER
4651
#include <intrin.h>
4752
#else

‎singleheader/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
c++ -O3 -std=c++17 -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
1+
c++ -O3 -std=c++17 -pthread -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson

‎singleheader/amalgamation_demo.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* auto-generated on Thu 07 Nov 2019 05:05:37 PM EST. Do not edit! */
1+
/* auto-generated on Wed Nov 20 11:15:43 EST 2019. Do not edit! */
22

33
#include <iostream>
44
#include "simdjson.h"

0 commit comments

Comments
 (0)