-
Notifications
You must be signed in to change notification settings - Fork 32
Expand file tree
/
Copy pathMultiVector.h
More file actions
172 lines (145 loc) · 4.86 KB
/
MultiVector.h
File metadata and controls
172 lines (145 loc) · 4.86 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
/**
* Copyright (c) 2017 HERE Europe B.V.
* See the LICENSE file in the root of this project for license details.
*/
#pragma once
#include "ExternalVector.h"
#include "MultiArrayView.h"
#include "internal/functional/Utility.h"
#include <cstdint>
#include <tuple>
namespace flatdata
{
/*
* Associates a list of heterogeneous data items with an index
* Usage:
* MultiVector< 32, A, C, B > vector = ...;
* for ( uint64_t index = 0; index < num_items; index++ )
* {
* auto list = vector.grow( );
* auto a = list.add< A >( );
* fill_data_a( a );
* auto another_a = list.add< A >( );
* fill_data_a( another_a );
* auto c = list.add< B >( );
* fill_data_b( b );
* auto d = list.add< C >( );
* fill_data_c( c );
* }
* if ( !vector.close( ) )
* {
* throw std::runtime_error( "Failed to write to disk" );
* }
* Internally data is stored like this:
* Index: Array< uint64_t > encodes start/end byte in Data array for each element i
* Data: ListBuilder of serialized (Tag,Type) tuples
*/
template < typename IndexType, typename... Args >
class MultiVector
{
public:
/// Class used to fill an individual list
class ListBuilder
{
public:
template < typename T >
typename T::MutatorType add( );
private:
friend class MultiVector;
std::vector< unsigned char >* m_data;
explicit ListBuilder( std::vector< unsigned char >* data );
};
public:
using View = MultiArrayView< IndexType, Args... >;
MultiVector( ExternalVector< IndexType > index, std::unique_ptr< ResourceHandle > data_handle );
ListBuilder grow( );
/**
* @brief Flushes remaining elements in buffer to disk and closes the multivector.
* After the multivector is closed, no elements can be added to it anymore.
* @return MultiArrayView to the written data. May fail, in this case the view is
* invalid (cf. bool operator of MultiArrayView).
*/
View close( );
public:
static_assert( all_true< ( !Args::IS_OVERLAPPING_WITH_NEXT )... >::value,
"Cannot use range/overlapping structs in MultiVector" );
private:
void add_to_index( );
void flush( );
private:
std::vector< unsigned char > m_data;
ExternalVector< IndexType > m_index;
std::unique_ptr< ResourceHandle > m_handle;
size_t m_size_flushed = 0;
};
// -------------------------------------------------------------------------------------------------
template < typename IndexType, typename... Args >
MultiVector< IndexType, Args... >::ListBuilder::ListBuilder( std::vector< unsigned char >* data )
: m_data( data )
{
}
template < typename IndexType, typename... Args >
template < typename T >
typename T::MutatorType
MultiVector< IndexType, Args... >::ListBuilder::add( )
{
static_assert(
index_of< typename T::AccessorType, typename Args::AccessorType... >::value < 256,
"Too many types or missing type" );
unsigned char index
= index_of< typename T::AccessorType, typename Args::AccessorType... >::value;
size_t old_size = m_data->size( );
size_t increment = 1 + T::size_in_bytes( );
m_data->resize( old_size + increment );
( *m_data )[ old_size - PADDING_SIZE ] = index;
return typename T::MutatorType( m_data->data( ) + 1 + old_size - PADDING_SIZE );
}
template < typename IndexType, typename... Args >
MultiVector< IndexType, Args... >::MultiVector( ExternalVector< IndexType > index,
std::unique_ptr< ResourceHandle > data_handle )
: m_index( std::move( index ) )
, m_handle( std::move( data_handle ) )
{
m_data.resize( PADDING_SIZE );
}
template < typename IndexType, typename... Args >
typename MultiVector< IndexType, Args... >::ListBuilder
MultiVector< IndexType, Args... >::grow( )
{
if ( m_data.size( ) > 1024 * 1024 * 32 )
{
flush( );
}
add_to_index( );
return ListBuilder{ &m_data };
}
template < typename IndexType, typename... Args >
typename MultiVector< IndexType, Args... >::View
MultiVector< IndexType, Args... >::close( )
{
add_to_index( ); // sentinel for last item
flush( );
MemoryDescriptor data = m_handle->close( );
ArrayView< IndexType > index_view = m_index.close( );
if ( !data || !index_view )
{
return { };
}
return { index_view, data.data( ) };
}
template < typename IndexType, typename... Args >
void
MultiVector< IndexType, Args... >::add_to_index( )
{
m_index.grow( ).value = m_size_flushed + m_data.size( ) - PADDING_SIZE;
}
template < typename IndexType, typename... Args >
void
MultiVector< IndexType, Args... >::flush( )
{
m_handle->write( m_data.data( ), m_data.size( ) - PADDING_SIZE );
m_size_flushed += m_data.size( ) - PADDING_SIZE;
m_data.resize( 0 );
m_data.resize( PADDING_SIZE );
}
} // namespace flatdata