From 3e851e961ed128e27641fe3151883c80df0cf59f Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Tue, 24 Mar 2020 11:44:28 +0300 Subject: [PATCH 001/606] (docs) Fix wrong output in `iterator` page. --- docs/source/quickref/iterator.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/quickref/iterator.rst b/docs/source/quickref/iterator.rst index 6961d8e3e..ec40f3e13 100644 --- a/docs/source/quickref/iterator.rst +++ b/docs/source/quickref/iterator.rst @@ -126,7 +126,7 @@ Iterating over axis 1: std::cout << *iter++ << std::endl; } // Prints: - // { 1, 5, 13 } + // { 1, 5, 9 } // { 2, 6, 10 } // { 3, 7, 11 } // { 4, 8, 12 } From 18cb139f1cba007785319ef25eea4cb211e712ce Mon Sep 17 00:00:00 2001 From: neok-m4700 Date: Thu, 26 Mar 2020 23:21:09 +0100 Subject: [PATCH 002/606] docs: add linear indexing example --- docs/source/numpy.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/numpy.rst b/docs/source/numpy.rst index 7657d9352..f1ad924e5 100644 --- a/docs/source/numpy.rst +++ b/docs/source/numpy.rst @@ -130,6 +130,9 @@ Slicing and indexing +=====================================================+=====================================================+ | ``a[3, 2]`` | ``a(3, 2)`` | +-----------------------------------------------------+-----------------------------------------------------+ +| ``a.flat[4]`` || ``a[4]`` | +| || ``a(4)`` | ++-----------------------------------------------------+-----------------------------------------------------+ | ``a[3]`` || ``xt::view(a, 3, xt::all())`` | | || ``xt::row(a, 3)`` | +-----------------------------------------------------+-----------------------------------------------------+ From f0179648bd9a9370bb7bfbaf1d918496c9754717 Mon Sep 17 00:00:00 2001 From: zhujun98 Date: Tue, 24 Mar 2020 21:48:48 +0100 Subject: [PATCH 003/606] Fix variance overload --- include/xtensor/xmath.hpp | 28 +++++++++++++++++++++++---- test/test_extended_xmath_reducers.cpp | 6 ++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index 9ee5d010e..93a911b34 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -2089,7 +2089,7 @@ namespace detail { return detail::mean_noaxis(square(abs(sc - mean(sc, es))), ddof, es); } - template )> inline auto variance(E&& e, EVS es = EVS()) { @@ -2143,7 +2143,7 @@ namespace detail { } template >, xtl::negation>>, xtl::negation>)> + XTL_REQUIRES(xtl::negation>, xtl::negation>>, is_reducer_options)> inline auto variance(E&& e, X&& axes, EVS es = EVS()) { return variance(std::forward(e), std::forward(axes), 0u, es); @@ -2182,13 +2182,23 @@ namespace detail { es); } - template + template )> inline auto variance(E&& e, const A (&axes)[N], EVS es = EVS()) { return variance(std::forward(e), xtl::forward_sequence, decltype(axes)>(axes), es); } + + template + inline auto variance(E&& e, const A (&axes)[N], D const& ddof, EVS es = EVS()) + { + return variance(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + ddof, + es); + } #else template inline auto stddev(E&& e, std::initializer_list axes, EVS es = EVS()) @@ -2198,13 +2208,23 @@ namespace detail { es); } - template + template )> inline auto variance(E&& e, std::initializer_list axes, EVS es = EVS()) { return variance(std::forward(e), xtl::forward_sequence, decltype(axes)>(axes), es); } + + template + inline auto variance(E&& e, std::initializer_list axes, D const& ddof, EVS es = EVS()) + { + return variance(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + ddof, + es); + } #endif /** diff --git a/test/test_extended_xmath_reducers.cpp b/test/test_extended_xmath_reducers.cpp index e7447803b..07acaeb99 100644 --- a/test/test_extended_xmath_reducers.cpp +++ b/test/test_extended_xmath_reducers.cpp @@ -1268,17 +1268,23 @@ namespace xt auto st_all = xt::stddev(py_a); auto vr_all = xt::variance(py_a); auto vr_all_ddof = xt::variance(py_a, {0, 1, 2, 3}, 1); + std::vector axes_all = {0, 1, 2, 3}; + auto vr_all_ddof_with_axes_arr = xt::variance(py_a, axes_all, 1); auto st = xt::stddev(py_a, {0, 2}); auto vr = xt::variance(py_a, {0, 2}); auto vr_ddof = xt::variance(py_a, {0, 2}, 1); + std::vector axes02 = {0, 2}; + auto vr_ddof_with_axes_arr = xt::variance(py_a, axes02, 1); EXPECT_TRUE(xt::allclose(st_all, py_st_all)); EXPECT_TRUE(xt::allclose(vr_all, py_vr_all)); EXPECT_TRUE(xt::allclose(vr_all_ddof, py_vr_all_ddof)); + EXPECT_TRUE(xt::allclose(vr_all_ddof_with_axes_arr, py_vr_all_ddof)); EXPECT_TRUE(xt::allclose(st, py_st)); EXPECT_TRUE(xt::allclose(vr, py_vr)); EXPECT_TRUE(xt::allclose(vr_ddof, py_vr_ddof)); + EXPECT_TRUE(xt::allclose(vr_ddof_with_axes_arr, py_vr_ddof)); } } From 2eb96663c95d9f1d90f73fe3a41120daa5d23ccd Mon Sep 17 00:00:00 2001 From: zhujun98 Date: Fri, 20 Mar 2020 23:23:55 +0100 Subject: [PATCH 004/606] Add result type template argument for stddev, variance, nanstd and nanvar --- include/xtensor/xmath.hpp | 133 +++++++++++++++++--------------- test/test_xmath_result_type.cpp | 53 +++++++++++-- test/test_xnan_functions.cpp | 67 ++++++++++++---- 3 files changed, 167 insertions(+), 86 deletions(-) diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index 93a911b34..4a9025341 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -2081,26 +2081,27 @@ namespace detail { } } - template , std::is_integral)> inline auto variance(E&& e, D const& ddof, EVS es = EVS()) { decltype(auto) sc = detail::shared_forward(e); - return detail::mean_noaxis(square(abs(sc - mean(sc, es))), ddof, es); + return detail::mean_noaxis(square(sc - mean(sc, es)), ddof, es); } - template )> inline auto variance(E&& e, EVS es = EVS()) { - return variance(std::forward(e), 0u, es); + return variance(std::forward(e), 0u, es); } - template )> inline auto stddev(E&& e, EVS es = EVS()) { - return sqrt(variance(std::forward(e), es)); + using result_type = std::conditional_t::value, double, T>; + return cast(sqrt(variance(std::forward(e), es))); } /** @@ -2121,7 +2122,7 @@ namespace detail { * * @sa stddev, mean */ - template >, std::is_integral)> inline auto variance(E&& e, X&& axes, D const& ddof, EVS es = EVS()) { @@ -2129,7 +2130,7 @@ namespace detail { // note: forcing copy of first axes argument -- is there a better solution? auto axes_copy = axes; // always eval to prevent repeated evaluations in the next calls - auto inner_mean = eval(mean(sc, std::move(axes_copy), evaluation_strategy::immediate)); + auto inner_mean = eval(mean(sc, std::move(axes_copy), evaluation_strategy::immediate)); // fake keep_dims = 1 auto keep_dim_shape = e.shape(); @@ -2139,14 +2140,14 @@ namespace detail { } auto mrv = reshape_view(std::move(inner_mean), std::move(keep_dim_shape)); - return detail::mean(square(abs(sc - std::move(mrv))), std::forward(axes), ddof, es); + return detail::mean(square(sc - std::move(mrv)), std::forward(axes), ddof, es); } - template >, xtl::negation>>, is_reducer_options)> inline auto variance(E&& e, X&& axes, EVS es = EVS()) { - return variance(std::forward(e), std::forward(axes), 0u, es); + return variance(std::forward(e), std::forward(axes), 0u, es); } /** @@ -2166,64 +2167,65 @@ namespace detail { * * @sa variance, mean */ - template >)> inline auto stddev(E&& e, X&& axes, EVS es = EVS()) { - return sqrt(variance(std::forward(e), std::forward(axes), es)); + using result_type = std::conditional_t::value, double, T>; + return cast(sqrt(variance(std::forward(e), std::forward(axes), es))); } #ifndef X_OLD_CLANG - template + template inline auto stddev(E&& e, const A (&axes)[N], EVS es = EVS()) { - return stddev(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return stddev(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } - template )> inline auto variance(E&& e, const A (&axes)[N], EVS es = EVS()) { - return variance(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return variance(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } - template + template inline auto variance(E&& e, const A (&axes)[N], D const& ddof, EVS es = EVS()) { - return variance(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - ddof, - es); + return variance(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + ddof, + es); } #else - template + template inline auto stddev(E&& e, std::initializer_list axes, EVS es = EVS()) { - return stddev(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return stddev(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } - template )> inline auto variance(E&& e, std::initializer_list axes, EVS es = EVS()) { - return variance(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return variance(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } - template + template inline auto variance(E&& e, std::initializer_list axes, D const& ddof, EVS es = EVS()) { - return variance(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - ddof, - es); + return variance(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + ddof, + es); } #endif @@ -2713,19 +2715,20 @@ namespace detail { } #endif - template )> inline auto nanvar(E&& e, EVS es = EVS()) { decltype(auto) sc = detail::shared_forward(e); - return nanmean(square(abs(sc - nanmean(sc))), es); + return nanmean(square(sc - nanmean(sc)), es); } - template )> inline auto nanstd(E&& e, EVS es = EVS()) { - return sqrt(nanvar(std::forward(e), es)); + using result_type = typename std::conditional_t::value, double, T>; + return cast(sqrt(nanvar(std::forward(e), es))); } /** @@ -2745,14 +2748,15 @@ namespace detail { * * @sa nanstd, nanmean */ - template >)> inline auto nanvar(E&& e, X&& axes, EVS es = EVS()) { decltype(auto) sc = detail::shared_forward(e); // note: forcing copy of first axes argument -- is there a better solution? auto axes_copy = axes; - auto inner_mean = nanmean(sc, std::move(axes_copy)); + using result_type = typename std::conditional_t::value, double, T>; + auto inner_mean = nanmean(sc, std::move(axes_copy)); // fake keep_dims = 1 auto keep_dim_shape = e.shape(); @@ -2761,7 +2765,7 @@ namespace detail { keep_dim_shape[el] = 1; } auto mrv = reshape_view(std::move(inner_mean), std::move(keep_dim_shape)); - return nanmean(square(abs(sc - std::move(mrv))), std::forward(axes), es); + return nanmean(square(sc - std::move(mrv)), std::forward(axes), es); } /** @@ -2781,44 +2785,45 @@ namespace detail { * * @sa nanvar, nanmean */ - template >)> inline auto nanstd(E&& e, X&& axes, EVS es = EVS()) { - return sqrt(nanvar(std::forward(e), std::forward(axes), es)); + using result_type = typename std::conditional_t::value, double, T>; + return cast(sqrt(nanvar(std::forward(e), std::forward(axes), es))); } #ifndef X_OLD_CLANG - template + template inline auto nanstd(E&& e, const A (&axes)[N], EVS es = EVS()) { - return nanstd(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return nanstd(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } - template + template inline auto nanvar(E&& e, const A (&axes)[N], EVS es = EVS()) { - return nanvar(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return nanvar(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } #else - template + template inline auto nanstd(E&& e, std::initializer_list axes, EVS es = EVS()) { - return nanstd(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return nanstd(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } - template + template inline auto nanvar(E&& e, std::initializer_list axes, EVS es = EVS()) { - return nanvar(std::forward(e), - xtl::forward_sequence, decltype(axes)>(axes), - es); + return nanvar(std::forward(e), + xtl::forward_sequence, decltype(axes)>(axes), + es); } #endif diff --git a/test/test_xmath_result_type.cpp b/test/test_xmath_result_type.cpp index 52e11ef00..afcea8dcd 100644 --- a/test/test_xmath_result_type.cpp +++ b/test/test_xmath_result_type.cpp @@ -66,21 +66,32 @@ namespace xt CHECK_RESULT_TYPE(FUNC(INPUT), signed int); \ CHECK_RESULT_TYPE(FUNC(INPUT), int); \ CHECK_RESULT_TYPE(FUNC(INPUT), unsigned long long); \ - CHECK_RESULT_TYPE(FUNC(INPUT), signed long long); + CHECK_RESULT_TYPE(FUNC(INPUT), signed long long); \ + CHECK_RESULT_TYPE(FUNC(INPUT), long long); \ + CHECK_RESULT_TYPE(FUNC(INPUT), float); \ + CHECK_RESULT_TYPE(FUNC(INPUT), double); TEST(xmath, result_type) { shape_type shape = {3, 2}; xarray auchar(shape); xarray ashort(shape); + xarray aushort(shape); xarray aint(shape); xarray auint(shape); + xarray along(shape); xarray aulong(shape); xarray afloat(shape); xarray adouble(shape); xarray> afcomplex(shape); xarray> adcomplex(shape); +#define CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(INPUT) \ + CHECK_TEMPLATED_RESULT_TYPE(mean, INPUT) \ + CHECK_TEMPLATED_RESULT_TYPE(variance, INPUT) +// FIXME: the first 6 checks in "#define CHECK_TEMPLATED_RESULT_TYPE(FUNC, INPUT)" fail +// CHECK_TEMPLATED_RESULT_TYPE(stddev, INPUT) + /***************** * unsigned char * *****************/ @@ -92,7 +103,7 @@ namespace xt CHECK_RESULT_TYPE(sum(auchar), unsigned long long); CHECK_RESULT_TYPE(mean(auchar), double); CHECK_RESULT_TYPE(minmax(auchar), ARRAY_TYPE(unsigned char)); - CHECK_TEMPLATED_RESULT_TYPE(mean, auchar); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(auchar); /********* * short * @@ -105,7 +116,20 @@ namespace xt CHECK_RESULT_TYPE(sum(ashort), long long); CHECK_RESULT_TYPE(mean(ashort), double); CHECK_RESULT_TYPE(minmax(ashort), ARRAY_TYPE(short)); - CHECK_TEMPLATED_RESULT_TYPE(mean, ashort); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(ashort); + + /****************** + * unsigned short * + ******************/ + CHECK_RESULT_TYPE(aushort + aushort, int); + CHECK_RESULT_TYPE(2u * aushort, unsigned int); + CHECK_RESULT_TYPE(2.0 * aushort, double); + CHECK_RESULT_TYPE(sqrt(aushort), double); + CHECK_RESULT_TYPE(abs(aushort), unsigned short); + CHECK_RESULT_TYPE(sum(aushort), unsigned long long); + CHECK_RESULT_TYPE(mean(aushort), double); + CHECK_RESULT_TYPE(minmax(aushort), ARRAY_TYPE(unsigned short)); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(ashort); /******* * int * @@ -118,7 +142,7 @@ namespace xt CHECK_RESULT_TYPE(sum(aint), long long); CHECK_RESULT_TYPE(mean(aint), double); CHECK_RESULT_TYPE(minmax(aint), ARRAY_TYPE(int)); - CHECK_TEMPLATED_RESULT_TYPE(mean, aint); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(aint); /**************** * unsigned int * @@ -131,7 +155,20 @@ namespace xt CHECK_RESULT_TYPE(sum(auint), unsigned long long); CHECK_RESULT_TYPE(mean(auint), double); CHECK_RESULT_TYPE(minmax(auint), ARRAY_TYPE(unsigned int)); - CHECK_TEMPLATED_RESULT_TYPE(mean, auint); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(auint); + + /********************** + * long long * + **********************/ + CHECK_RESULT_TYPE(along + along, signed long long); + CHECK_RESULT_TYPE(2 * along, signed long long); + CHECK_RESULT_TYPE(2.0 * along, double); + CHECK_RESULT_TYPE(sqrt(along), double); + CHECK_RESULT_TYPE(abs(along), signed long long); + CHECK_RESULT_TYPE(sum(along), signed long long); + CHECK_RESULT_TYPE(mean(along), double); + CHECK_RESULT_TYPE(minmax(along), ARRAY_TYPE(signed long long)); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(along); /********************** * unsigned long long * @@ -144,7 +181,7 @@ namespace xt CHECK_RESULT_TYPE(sum(aulong), unsigned long long); CHECK_RESULT_TYPE(mean(aulong), double); CHECK_RESULT_TYPE(minmax(aulong), ARRAY_TYPE(unsigned long long)); - CHECK_TEMPLATED_RESULT_TYPE(mean, aulong); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(aulong); /********* * float * @@ -157,7 +194,7 @@ namespace xt CHECK_RESULT_TYPE(sum(afloat), double); CHECK_RESULT_TYPE(mean(afloat), double); CHECK_RESULT_TYPE(minmax(afloat), ARRAY_TYPE(float)); - CHECK_TEMPLATED_RESULT_TYPE(mean, afloat); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(afloat); /********** * double * @@ -169,7 +206,7 @@ namespace xt CHECK_RESULT_TYPE(sum(adouble), double); CHECK_RESULT_TYPE(mean(adouble), double); CHECK_RESULT_TYPE(minmax(adouble), ARRAY_TYPE(double)); - CHECK_TEMPLATED_RESULT_TYPE(mean, adouble); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(adouble); /*********************** * std::complex * diff --git a/test/test_xnan_functions.cpp b/test/test_xnan_functions.cpp index dc3cb89fa..ded72ea60 100644 --- a/test/test_xnan_functions.cpp +++ b/test/test_xnan_functions.cpp @@ -202,34 +202,73 @@ namespace xt TEST(xnanfunctions, result_type) { shape_type shape = {4, 3, 2}; + xarray ashort(shape); + xarray aushort(shape); xarray aint(shape); + xarray auint(shape); + xarray along(shape); + xarray aulong(shape); xarray afloat(shape); xarray adouble(shape); +#define CHECK_RESULT_TYPE_FOR_ALL(INPUT, RESULT_TYPE) \ + CHECK_RESULT_TYPE(nansum(INPUT, {1, 2}), RESULT_TYPE); \ + CHECK_RESULT_TYPE(nanmean(INPUT, {1, 2}), double); \ + CHECK_RESULT_TYPE(nanvar(INPUT, {1, 2}), double); \ + CHECK_RESULT_TYPE(nanstd(INPUT, {1, 2}), double); + +#define CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(INPUT, RESULT_TYPE) \ + CHECK_RESULT_TYPE(nansum(INPUT, {1, 2}), RESULT_TYPE) \ + CHECK_RESULT_TYPE(nanmean(INPUT, {1, 2}), RESULT_TYPE) \ + CHECK_RESULT_TYPE(nanvar(INPUT, {1, 2}), RESULT_TYPE) \ + CHECK_RESULT_TYPE(nanstd(INPUT, {1, 2}), RESULT_TYPE) + + /********* + * short * + *********/ + CHECK_RESULT_TYPE_FOR_ALL(ashort, short); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(ashort, int); + + /****************** + * unsigned short * + ******************/ + CHECK_RESULT_TYPE_FOR_ALL(aushort, unsigned short); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(aushort, unsigned int); + /********* * int * *********/ - CHECK_RESULT_TYPE(nansum(aint, {1, 2}), int); - CHECK_RESULT_TYPE(nanmean(aint, {1, 2}), double); - CHECK_RESULT_TYPE(nanstd(aint, {1, 2}), double); - CHECK_RESULT_TYPE(nanvar(aint, {1, 2}), double); - CHECK_RESULT_TYPE(nanmean(aint, {1, 2}), int); + CHECK_RESULT_TYPE_FOR_ALL(aint, int); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(aint, int); + + /**************** + * unsigned int * + ****************/ + CHECK_RESULT_TYPE_FOR_ALL(auint, unsigned int); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(auint, unsigned int); + + /********************** + * long long * + **********************/ + CHECK_RESULT_TYPE_FOR_ALL(along, signed long long); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(along, signed long long); + + /********************** + * unsigned long long * + **********************/ + CHECK_RESULT_TYPE_FOR_ALL(aulong, unsigned long long); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(aulong, unsigned long long); /********* * float * *********/ - CHECK_RESULT_TYPE(nansum(afloat, {1, 2}), float); - CHECK_RESULT_TYPE(nanmean(afloat, {1, 2}), double); - CHECK_RESULT_TYPE(nanstd(afloat, {1, 2}), double); - CHECK_RESULT_TYPE(nanvar(afloat, {1, 2}), double); - CHECK_RESULT_TYPE(nanmean(afloat, {1, 2}), float); + CHECK_RESULT_TYPE_FOR_ALL(afloat, float); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(afloat, float); /********** * double * **********/ - CHECK_RESULT_TYPE(nansum(adouble, {1, 2}), double); - CHECK_RESULT_TYPE(nanmean(adouble, {1, 2}), double); - CHECK_RESULT_TYPE(nanstd(adouble, {1, 2}), double); - CHECK_RESULT_TYPE(nanvar(adouble, {1, 2}), double); + CHECK_RESULT_TYPE_FOR_ALL(adouble, double); + CHECK_TEMPLATED_RESULT_TYPE_FOR_ALL(adouble, double); } } From 7606c567fad606c3dc6e06a4d7b9a76c8555022b Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 31 Mar 2020 09:27:15 +0200 Subject: [PATCH 005/606] Added missing header to CMakeLists.txt --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 15a758873..ffe5ee863 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,6 +119,7 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xarray.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xassign.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xaxis_iterator.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/xaxis_slice_iterator.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xbroadcast.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xbuffer_adaptor.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xbuilder.hpp From 495433834fb1d62997cced9b04d8bec19017821b Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 31 Mar 2020 22:52:10 +0200 Subject: [PATCH 006/606] Fixed xview on const keep and const drop slices --- docs/source/quickref/basic.rst | 2 +- include/xtensor/xview.hpp | 3 ++- test/test_xview.cpp | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/source/quickref/basic.rst b/docs/source/quickref/basic.rst index aaa8bc213..5aa105dd0 100644 --- a/docs/source/quickref/basic.rst +++ b/docs/source/quickref/basic.rst @@ -125,7 +125,7 @@ length of the underlying buffer and remaining dimensions: std::cout << a0 << std::endl; // outputs {{1., 2., 3.}, {4., 5., 6. }} - xt::xtensor a1 = {{1. 2.}, {3., 4.}, {5., 6.}}; + xt::xtensor a1 = {{1., 2.}, {3., 4.}, {5., 6.}}; a1.reshape({-1, 3}); std::cout << a1 << std::endl; // outputs {{1., 2., 3.}, {4., 5., 6. }} diff --git a/include/xtensor/xview.hpp b/include/xtensor/xview.hpp index 52e3837ba..911b84d05 100644 --- a/include/xtensor/xview.hpp +++ b/include/xtensor/xview.hpp @@ -158,7 +158,8 @@ namespace xt // If we have no discontiguous slices, we can calculate strides for this view. template struct is_strided_view - : std::integral_constant, is_strided_slice_impl...>::value> + : std::integral_constant, + is_strided_slice_impl>...>::value> { }; diff --git a/test/test_xview.cpp b/test/test_xview.cpp index f35e4344e..9120ea983 100644 --- a/test/test_xview.cpp +++ b/test/test_xview.cpp @@ -1120,6 +1120,20 @@ namespace xt EXPECT_EQ(v1, exp_v1); } + TEST(xview, const_keep_drop_slice) + { + xt::xtensor xs = xt::arange(10); + const auto kidx = xt::keep(0, 3, 5); + const auto didx = xt::drop(1, 2, 4, 6, 7, 8, 9); + auto kv = xt::view(xs, kidx); + auto dv = xt::view(xs, didx); + xt::xtensor kres = kv; + xt::xtensor dres = dv; + xt::xtensor expected = { 0., 3., 5. }; + EXPECT_EQ(kres, expected); + EXPECT_EQ(dres, expected); + } + TEST(xview, mixed_types) { xt::xarray input; From 42f296e8c96a42317b2bdf15ec11209b4d379858 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 1 Apr 2020 16:01:02 +0200 Subject: [PATCH 007/606] Improved reducer function documentation --- docs/source/api/reducing_functions.rst | 26 +++++++++++++++++++++++++- docs/source/pitfall.rst | 21 +++++++++++++++++++++ include/xtensor/xmath.hpp | 3 +-- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/source/api/reducing_functions.rst b/docs/source/api/reducing_functions.rst index 4709d9573..643cf7b13 100644 --- a/docs/source/api/reducing_functions.rst +++ b/docs/source/api/reducing_functions.rst @@ -11,22 +11,40 @@ Reducing functions Defined in ``xtensor/xmath.hpp`` +.. doxygenfunction:: sum(E&&, EVS) + :project: xtensor + .. _sum-function-reference: .. doxygenfunction:: sum(E&&, X&&, EVS) :project: xtensor +.. doxygenfunction:: prod(E&&, EVS) + :project: xtensor + .. _prod-function-reference: .. doxygenfunction:: prod(E&&, X&&, EVS) :project: xtensor +.. doxygenfunction:: mean(E&&, EVS) + :project: xtensor + .. _mean-function-reference: .. doxygenfunction:: mean(E&&, X&&, EVS) :project: xtensor -.. _variance-function-reference: +.. doxygenfunction:: variance(E&&, EVS) + :project: xtensor + .. doxygenfunction:: variance(E&&, X&&, EVS) :project: xtensor +.. _variance-function-reference: +.. doxygenfunction:: variance(E&&, X&&, const D&, EVS) + :project: xtensor + +.. doxygenfunction:: stddev(E&&, EVS) + :project: xtensor + .. _stddev-function-reference: .. doxygenfunction:: stddev(E&&, X&&, EVS) :project: xtensor @@ -35,10 +53,16 @@ Defined in ``xtensor/xmath.hpp`` .. doxygenfunction:: diff(const xexpression&, unsigned int, std::ptrdiff_t) :project: xtensor +.. doxygenfunction:: amax(E&&, EVS) + :project: xtensor + .. _amax-function-reference: .. doxygenfunction:: amax(E&&, X&&, EVS) :project: xtensor +.. doxygenfunction:: amin(E&&, EVS) + :project: xtensor + .. _amin-function-reference: .. doxygenfunction:: amin(E&&, X&&, EVS) :project: xtensor diff --git a/docs/source/pitfall.rst b/docs/source/pitfall.rst index bac01ccea..32e2423a5 100644 --- a/docs/source/pitfall.rst +++ b/docs/source/pitfall.rst @@ -94,3 +94,24 @@ like so: auto xr2 = eval(xt::random::rand({10, 10})); // now xr(0, 0) == xr(0, 0) is true. + +variance arguments +------------------ + +When ``variance`` is passed an expression and an integer parameter, this latter +is the axis along which the variance must be computed, but the degree of freedom: + +.. code:: + + xt::xtensor a = {{1., 2., 3.}, {4., 5., 6.}}; + std::cout << xt::variance(a, 1) << std::endl; + // Outputs 3.5 + +If you want to specify an axis, you need to pass an initializer list: + +.. code:: + + xt::xtensor a = {{1., 2., 3.}, {4., 5., 6.}}; + std::cout << xt::variance(a, {1}) << std::endl; + .. Outputs { 0.666667, 0.666667 } + diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index 4a9025341..13e1bad27 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -1852,7 +1852,6 @@ namespace detail { * Reducing functions * **********************/ - /** * @defgroup red_functions reducing functions */ @@ -2124,7 +2123,7 @@ namespace detail { */ template >, std::is_integral)> - inline auto variance(E&& e, X&& axes, D const& ddof, EVS es = EVS()) + inline auto variance(E&& e, X&& axes, const D& ddof, EVS es = EVS()) { decltype(auto) sc = detail::shared_forward(e); // note: forcing copy of first axes argument -- is there a better solution? From 2038c99d7dbe386e69bcdb18ba3e86b0706676be Mon Sep 17 00:00:00 2001 From: neok-m4700 Date: Fri, 3 Apr 2020 19:19:53 +0200 Subject: [PATCH 008/606] Documentation typo --- include/xtensor/xstrided_view.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xstrided_view.hpp b/include/xtensor/xstrided_view.hpp index b394898bc..7e62da3f3 100644 --- a/include/xtensor/xstrided_view.hpp +++ b/include/xtensor/xstrided_view.hpp @@ -641,7 +641,7 @@ namespace xt * * \code{.cpp} * xt::xarray a = {{1, 2, 3}, {4, 5, 6}}; - * xt::slice_vector sv({xt::range(0, 1)}); + * xt::xstrided_slice_vector sv({xt::range(0, 1)}); * sv.push_back(xt::range(0, 3, 2)); * auto v = xt::strided_view(a, sv); * // ==> {{1, 3}} From fb316664faada67a633c5be4aa406a3ff205c00f Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 3 Apr 2020 21:52:03 +0200 Subject: [PATCH 009/606] Added static_assert to adapt methods --- include/xtensor/xadapt.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/xtensor/xadapt.hpp b/include/xtensor/xadapt.hpp index 942bb7fc9..659114124 100644 --- a/include/xtensor/xadapt.hpp +++ b/include/xtensor/xadapt.hpp @@ -74,6 +74,7 @@ namespace xt inline xarray_adaptor, L, std::decay_t> adapt(C&& container, const SC& shape, layout_type l = L) { + static_assert(!std::is_integral::value, "shape cannot be a integer"); using return_type = xarray_adaptor, L, std::decay_t>; return return_type(std::forward(container), shape, l); } @@ -89,6 +90,7 @@ namespace xt std::is_pointer)> inline auto adapt(C&& pointer, const SC& shape, layout_type l = L) { + static_assert(!std::is_integral::value, "shape cannot be a integer"); using buffer_type = xbuffer_adaptor>; using return_type = xarray_adaptor>; std::size_t size = compute_size(shape); @@ -108,6 +110,7 @@ namespace xt inline xarray_adaptor, layout_type::dynamic, std::decay_t> adapt(C&& container, SC&& shape, SS&& strides) { + static_assert(!std::is_integral>::value, "shape cannot be a integer"); using return_type = xarray_adaptor, layout_type::dynamic, std::decay_t>; return return_type(std::forward(container), xtl::forward_sequence(shape), @@ -130,6 +133,7 @@ namespace xt inline xarray_adaptor, O, A>, L, SC> adapt(P&& pointer, typename A::size_type size, O ownership, const SC& shape, layout_type l = L, const A& alloc = A()) { + static_assert(!std::is_integral::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; using return_type = xarray_adaptor; @@ -154,6 +158,7 @@ namespace xt inline xarray_adaptor, O, A>, layout_type::dynamic, std::decay_t> adapt(P&& pointer, typename A::size_type size, O ownership, SC&& shape, SS&& strides, const A& alloc = A()) { + static_assert(!std::is_integral>::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; using return_type = xarray_adaptor>; @@ -226,6 +231,7 @@ namespace xt inline xtensor_adaptor::value, L> adapt(C&& container, const SC& shape, layout_type l = L) { + static_assert(!std::is_integral::value, "shape cannot be a integer"); constexpr std::size_t N = detail::array_size::value; using return_type = xtensor_adaptor, N, L>; return return_type(std::forward(container), shape, l); @@ -242,6 +248,7 @@ namespace xt std::is_pointer)> inline auto adapt(C&& pointer, const SC& shape, layout_type l = L) { + static_assert(!std::is_integral::value, "shape cannot be a integer"); using buffer_type = xbuffer_adaptor>; constexpr std::size_t N = detail::array_size::value; using return_type = xtensor_adaptor; @@ -261,6 +268,7 @@ namespace xt inline xtensor_adaptor::value, layout_type::dynamic> adapt(C&& container, SC&& shape, SS&& strides) { + static_assert(!std::is_integral>::value, "shape cannot be a integer"); constexpr std::size_t N = detail::array_size::value; using return_type = xtensor_adaptor, N, layout_type::dynamic>; return return_type(std::forward(container), @@ -306,6 +314,7 @@ namespace xt inline xtensor_adaptor, O, A>, detail::array_size::value, L> adapt(P&& pointer, typename A::size_type size, O ownership, const SC& shape, layout_type l = L, const A& alloc = A()) { + static_assert(!std::is_integral::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; constexpr std::size_t N = detail::array_size::value; @@ -331,6 +340,7 @@ namespace xt inline xtensor_adaptor, O, A>, detail::array_size::value, layout_type::dynamic> adapt(P&& pointer, typename A::size_type size, O ownership, SC&& shape, SS&& strides, const A& alloc = A()) { + static_assert(!std::is_integral>::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; constexpr std::size_t N = detail::array_size::value; From eb3c54b6d0c04d19359b8706a9cc6e6288579939 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 10 Apr 2020 23:33:17 +0200 Subject: [PATCH 010/606] Removed allocator deprecated calls --- include/xtensor/xbuffer_adaptor.hpp | 55 +++++++++++++++-------------- include/xtensor/xstorage.hpp | 3 +- include/xtensor/xutils.hpp | 7 ++-- 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/include/xtensor/xbuffer_adaptor.hpp b/include/xtensor/xbuffer_adaptor.hpp index 852dc3f6d..2ce4a800b 100644 --- a/include/xtensor/xbuffer_adaptor.hpp +++ b/include/xtensor/xbuffer_adaptor.hpp @@ -51,17 +51,18 @@ namespace xt using self_type = xbuffer_storage; using allocator_type = A; using destructor_type = allocator_type; - using value_type = typename allocator_type::value_type; + using allocator_traits = std::allocator_traits; + using value_type = typename allocator_traits::value_type; using reference = std::conditional_t>>::value, - typename allocator_type::const_reference, - typename allocator_type::reference>; - using const_reference = typename allocator_type::const_reference; + const value_type&, + value_type&>; + using const_reference = const value_type&; using pointer = std::conditional_t>>::value, - typename allocator_type::const_pointer, - typename allocator_type::pointer>; - using const_pointer = typename allocator_type::const_pointer; - using size_type = typename allocator_type::size_type; - using difference_type = typename allocator_type::difference_type; + typename allocator_traits::const_pointer, + typename allocator_traits::pointer>; + using const_pointer = typename allocator_traits::const_pointer; + using size_type = typename allocator_traits::size_type; + using difference_type = typename allocator_traits::difference_type; xbuffer_storage(); @@ -91,16 +92,17 @@ namespace xt using destructor_type = D; using value_type = std::remove_const_t>>; using allocator_type = std::allocator; + using allocator_traits = std::allocator_traits; using reference = std::conditional_t>>::value, - typename allocator_type::const_reference, - typename allocator_type::reference>; - using const_reference = typename allocator_type::const_reference; + const value_type&, + value_type&>; + using const_reference = const value_type&; using pointer = std::conditional_t>>::value, - typename allocator_type::const_pointer, - typename allocator_type::pointer>; - using const_pointer = typename allocator_type::const_pointer; - using size_type = typename allocator_type::size_type; - using difference_type = typename allocator_type::difference_type; + typename allocator_traits::const_pointer, + typename allocator_traits::pointer>; + using const_pointer = typename allocator_traits::const_pointer; + using size_type = typename allocator_traits::size_type; + using difference_type = typename allocator_traits::difference_type; xbuffer_smart_pointer(); @@ -130,17 +132,18 @@ namespace xt using self_type = xbuffer_owner_storage; using allocator_type = A; using destructor_type = allocator_type; - using value_type = typename allocator_type::value_type; + using allocator_traits = std::allocator_traits; + using value_type = typename allocator_traits::value_type; using reference = std::conditional_t>>::value, - typename allocator_type::const_reference, - typename allocator_type::reference>; - using const_reference = typename allocator_type::const_reference; + const value_type&, + value_type&>; + using const_reference = const value_type&; using pointer = std::conditional_t>>::value, - typename allocator_type::const_pointer, - typename allocator_type::pointer>; - using const_pointer = typename allocator_type::const_pointer; - using size_type = typename allocator_type::size_type; - using difference_type = typename allocator_type::difference_type; + typename allocator_traits::const_pointer, + typename allocator_traits::pointer>; + using const_pointer = typename allocator_traits::const_pointer; + using size_type = typename allocator_traits::size_type; + using difference_type = typename allocator_traits::difference_type; xbuffer_owner_storage() = default; diff --git a/include/xtensor/xstorage.hpp b/include/xtensor/xstorage.hpp index 838c08f96..f702bee30 100644 --- a/include/xtensor/xstorage.hpp +++ b/include/xtensor/xstorage.hpp @@ -1329,7 +1329,8 @@ namespace xt template struct rebind_container> { - using allocator = typename A::template rebind::other; + using traits = std::allocator_traits; + using allocator = typename traits::template rebind_alloc; using type = svector; }; diff --git a/include/xtensor/xutils.hpp b/include/xtensor/xutils.hpp index ce3ce72f8..5a3f31bce 100644 --- a/include/xtensor/xutils.hpp +++ b/include/xtensor/xutils.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -736,7 +737,8 @@ namespace xt template struct rebind { - using other = tracking_allocator::other, P>; + using traits = std::allocator_traits; + using other = tracking_allocator, P>; }; }; @@ -774,7 +776,8 @@ namespace xt template class C, class T, class A> struct rebind_container> { - using allocator = typename A::template rebind::other; + using traits = std::allocator_traits; + using allocator = typename traits::template rebind_alloc; using type = C; }; From 814038ec48a46281646f797ae6d08f56ab6e881b Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Sun, 12 Apr 2020 23:04:52 +0200 Subject: [PATCH 011/606] Added missing overload of push_back --- include/xtensor/xstorage.hpp | 11 +++++++++++ test/test_xstorage.cpp | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/include/xtensor/xstorage.hpp b/include/xtensor/xstorage.hpp index f702bee30..c752159bc 100644 --- a/include/xtensor/xstorage.hpp +++ b/include/xtensor/xstorage.hpp @@ -677,6 +677,7 @@ namespace xt const_pointer data() const; void push_back(const T& elt); + void push_back(T&& elt); void pop_back(); iterator begin(); @@ -985,6 +986,16 @@ namespace xt *(m_end++) = elt; } + template + void svector::push_back(T&& elt) + { + if (m_end >= m_capacity) + { + grow(); + } + *(m_end++) = std::move(elt); + } + template void svector::pop_back() { diff --git a/test/test_xstorage.cpp b/test/test_xstorage.cpp index dd5a216c3..91a9f1ac8 100644 --- a/test/test_xstorage.cpp +++ b/test/test_xstorage.cpp @@ -107,15 +107,21 @@ namespace xt v2.erase(v2.begin() + 1, v2.end()); EXPECT_TRUE(std::equal(s2.begin(), s2.end(), v2.begin())); + std::size_t i1 = 50; + std::size_t i2 = 50; EXPECT_TRUE(s2.on_stack()); s2.push_back(10); s2.push_back(20); s2.push_back(30); s2.push_back(40); + s2.push_back(i1); + s2.push_back(std::move(i1)); v2.push_back(10); v2.push_back(20); v2.push_back(30); v2.push_back(40); + v2.push_back(i2); + v2.push_back(std::move(i2)); EXPECT_FALSE(s2.on_stack()); EXPECT_TRUE(std::equal(s2.begin(), s2.end(), v2.begin())); } From c2b3a686f61e500887ec40da899a7140bca870c4 Mon Sep 17 00:00:00 2001 From: Andrew Clifton Date: Mon, 13 Apr 2020 15:39:47 -0400 Subject: [PATCH 012/606] Initialized all members of xfunction_cache_impl to avoid sanitizer errors. --- include/xtensor/xfunction.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xfunction.hpp b/include/xtensor/xfunction.hpp index c0091547b..dabd2a7e1 100644 --- a/include/xtensor/xfunction.hpp +++ b/include/xtensor/xfunction.hpp @@ -50,7 +50,7 @@ namespace xt bool is_trivial; bool is_initialized; - xfunction_cache_impl() : shape(xtl::make_sequence(0, std::size_t(0))), is_initialized(false) {} + xfunction_cache_impl() : shape(xtl::make_sequence(0, std::size_t(0))), is_initialized(false), is_trivial(false) {} }; template From 1651d60d09fdf50e4509a1c8af6f0a2fded0c6d4 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 14 Apr 2020 11:13:20 +0200 Subject: [PATCH 013/606] Release 0.21.5 --- README.md | 6 +----- docs/source/changelog.rst | 29 +++++++++++++++++++++++++++++ environment.yml | 2 +- include/xtensor/xtensor_config.hpp | 2 +- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bde02af68..8fa618da7 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| | master | ^0.6.12 | ^7.4.6 | +| 0.21.5 | ^0.6.12 | ^7.4.6 | | 0.21.4 | ^0.6.12 | ^7.4.6 | | 0.21.3 | ^0.6.9 | ^7.4.4 | | 0.21.2 | ^0.6.9 | ^7.4.4 | @@ -110,11 +111,6 @@ library: | 0.20.2 | ^0.6.1 | ^7.0.0 | | 0.20.1 | ^0.6.1 | ^7.0.0 | | 0.20.0 | ^0.6.1 | ^7.0.0 | -| 0.19.4 | ^0.5.3 | ^7.0.0 | -| 0.19.3 | ^0.5.3 | ^7.0.0 | -| 0.19.2 | ^0.5.3 | ^7.0.0 | -| 0.19.1 | ^0.5.1 | ^7.0.0 | -| 0.19.0 | ^0.5.1 | ^7.0.0 | The dependency on `xsimd` is required if you want to enable SIMD acceleration in `xtensor`. This can be done by defining the macro `XTENSOR_USE_XSIMD` diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 1ff61b0fc..b456e6f28 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -7,6 +7,35 @@ Changelog ========= +0.21.5 +------ + +- Fix segfault when using ``xt::drop`` on an empty list of indices + `#1990 `_ +- Implemented missing methods in ``xrepeat`` class + `#1993 `_ +- Added extension base to ``xrepeat`` and clean up ``xbroadcast`` + `#1994 `_ +- Fix return type of ``nanmean`` and add unittest + `#1996 `_ +- Add result type template argument for ``stddev``, ``variance``, ``nanstd`` and ``nanvar`` + `#1999 `_ +- Fix variance overload + `#2002 `_ +- Added missing ``xaxis_slice_iterator`` header to CMakeLists.txt + `#2009 `_ +- Fixed xview on const keep and const drop slices + `#2010 `_ +- Added ``static_assert`` to ``adapt`` methods + `#2015 `_ +- Removed allocator deprecated calls + `#2018 `_ +- Added missing overload of ``push_back`` to ``svector`` + `#2024 `_ +- Initialized all members of ``xfunciton_cache_impl`` + `#2026 `_ + + 0.21.4 ------ diff --git a/environment.yml b/environment.yml index 568c2a764..ce388ff0a 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xtensor channels: - conda-forge dependencies: - - xtensor=0.21.4 + - xtensor=0.21.5 - xtensor-blas=0.17.2 - xeus-cling=0.8.1 - blas * *openblas" diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index e615f5173..26cb4ccc6 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -12,7 +12,7 @@ #define XTENSOR_VERSION_MAJOR 0 #define XTENSOR_VERSION_MINOR 21 -#define XTENSOR_VERSION_PATCH 5-dev +#define XTENSOR_VERSION_PATCH 5 // DETECT 3.6 <= clang < 3.8 for compiler bug workaround. #ifdef __clang__ From f87154ddd8c524cc91c346032ec52087452aa548 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 14 Apr 2020 11:14:39 +0200 Subject: [PATCH 014/606] Starting 0.21.6-dev --- include/xtensor/xtensor_config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index 26cb4ccc6..271cfd1a2 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -12,7 +12,7 @@ #define XTENSOR_VERSION_MAJOR 0 #define XTENSOR_VERSION_MINOR 21 -#define XTENSOR_VERSION_PATCH 5 +#define XTENSOR_VERSION_PATCH 6-dev // DETECT 3.6 <= clang < 3.8 for compiler bug workaround. #ifdef __clang__ From 1a81770b4d5f138f78f21085a9bee8ab2eeedc8c Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 16 Apr 2020 18:25:38 +0200 Subject: [PATCH 015/606] Removing unused parameter --- include/xtensor/xrepeat.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/xtensor/xrepeat.hpp b/include/xtensor/xrepeat.hpp index cdd6ad64b..ab1c0d832 100644 --- a/include/xtensor/xrepeat.hpp +++ b/include/xtensor/xrepeat.hpp @@ -21,7 +21,7 @@ namespace xt { template class xrepeat; - + template class xrepeat_stepper; @@ -153,7 +153,7 @@ namespace xt template bool has_linear_assign(const S& strides) const noexcept; - + const_stepper stepper_begin() const; const_stepper stepper_begin(const shape_type& s) const; @@ -384,7 +384,7 @@ namespace xt */ template template - inline bool xrepeat::broadcast_shape(S& shape, bool reuse_cache) const + inline bool xrepeat::broadcast_shape(S& shape, bool) const { return xt::broadcast_shape(m_shape, shape); } From e4f2bc5b596acf28aa25aa01752f6aa74a3222f1 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 16 Apr 2020 18:30:05 +0200 Subject: [PATCH 016/606] Fixing compiler warning --- include/xtensor/xfunction.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xtensor/xfunction.hpp b/include/xtensor/xfunction.hpp index dabd2a7e1..702ead106 100644 --- a/include/xtensor/xfunction.hpp +++ b/include/xtensor/xfunction.hpp @@ -50,7 +50,7 @@ namespace xt bool is_trivial; bool is_initialized; - xfunction_cache_impl() : shape(xtl::make_sequence(0, std::size_t(0))), is_initialized(false), is_trivial(false) {} + xfunction_cache_impl() : shape(xtl::make_sequence(0, std::size_t(0))), is_trivial(false), is_initialized(false) {} }; template @@ -152,7 +152,7 @@ namespace xt has_simd_interface, T>...> { }; - + /************* * xfunction * *************/ From 0805f0db44edd52fee9f373b88d40682b22de4e1 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 16 Apr 2020 22:42:58 +0200 Subject: [PATCH 017/606] Fixed azure-pipelines --- .azure-pipelines/azure-pipelines-osx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index bc9187ecf..282a8acfd 100644 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -17,7 +17,7 @@ jobs: echo "Removing homebrew for Azure to avoid conflicts with conda" curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew chmod +x ~/uninstall_homebrew - ~/uninstall_homebrew -fq + ~/uninstall_homebrew -f -q displayName: Remove homebrew - bash: | From 0df4cd152f621d8f5c4230929bbb7c808efcbef1 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Sun, 12 Apr 2020 12:26:23 +0200 Subject: [PATCH 018/606] Adding isin and in1d --- CMakeLists.txt | 1 + docs/source/numpy.rst | 4 + include/xtensor/xoperation.hpp | 1 + include/xtensor/xset_operation.hpp | 140 +++++++++++++++++++++++++++++ include/xtensor/xutils.hpp | 21 +++++ test/CMakeLists.txt | 1 + test/test_xoperation.cpp | 1 + test/test_xset_operation.cpp | 39 ++++++++ 8 files changed, 208 insertions(+) create mode 100644 include/xtensor/xset_operation.hpp create mode 100644 test/test_xset_operation.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ffe5ee863..a90850c40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,6 +163,7 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xrepeat.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xscalar.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xsemantic.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/xset_operation.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xshape.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xslice.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xsort.hpp diff --git a/docs/source/numpy.rst b/docs/source/numpy.rst index f1ad924e5..36d1cc723 100644 --- a/docs/source/numpy.rst +++ b/docs/source/numpy.rst @@ -299,6 +299,10 @@ where ``condition`` is falsy, and it does not evaluate ``b`` where ``condition`` +-----------------------------------------------+-----------------------------------------------+ | ``np.all(a)`` | ``xt::all(a)`` | +-----------------------------------------------+-----------------------------------------------+ +| ``np.isin(a, b)`` | ``xt::isin(a, b)`` | ++-----------------------------------------------+-----------------------------------------------+ +| ``np.in1d(a, b)`` | ``xt::in1d(a, b)`` | ++-----------------------------------------------+-----------------------------------------------+ | ``np.logical_and(a, b)`` | ``a && b`` | +-----------------------------------------------+-----------------------------------------------+ | ``np.logical_or(a, b)`` | ``a || b`` | diff --git a/include/xtensor/xoperation.hpp b/include/xtensor/xoperation.hpp index 4c8018f66..df18aa973 100644 --- a/include/xtensor/xoperation.hpp +++ b/include/xtensor/xoperation.hpp @@ -959,6 +959,7 @@ namespace xt { return detail::make_xfunction::functor>(std::forward(e)); } + } #endif diff --git a/include/xtensor/xset_operation.hpp b/include/xtensor/xset_operation.hpp new file mode 100644 index 000000000..9488054d8 --- /dev/null +++ b/include/xtensor/xset_operation.hpp @@ -0,0 +1,140 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XTENSOR_XSET_OPERATION_HPP +#define XTENSOR_XSET_OPERATION_HPP + +#include +#include +#include + +#include + +#include "xfunction.hpp" +#include "xutils.hpp" +#include "xscalar.hpp" +#include "xstrides.hpp" +#include "xstrided_view.hpp" +#include "xmath.hpp" + +namespace xt +{ + + /** + * @ingroup logical_operators + * @brief isin + * + * Returns a boolean array of the same shape as ``element`` that is ``true`` where an element of + * ``element`` is in ``test_elements`` and ``False`` otherwise. + * @param element an \ref xexpression + * @param test_elements an array + * @return a boolean array + */ + template + inline auto isin(E&& element, std::initializer_list test_elements) noexcept + { + auto lambda = [test_elements](const auto& t) { + return std::find(test_elements.begin(), test_elements.end(), t) != test_elements.end(); }; + return make_lambda_xfunction(std::move(lambda), std::forward(element)); + } + + /** + * @ingroup logical_operators + * @brief isin + * + * Returns a boolean array of the same shape as ``element`` that is ``true`` where an element of + * ``element`` is in ``test_elements`` and ``False`` otherwise. + * @param element an \ref xexpression + * @param test_elements an array + * @return a boolean array + */ + template ::value>> + inline auto isin(E&& element, F&& test_elements) noexcept + { + auto lambda = [&test_elements](const auto& t) { + return std::find(test_elements.begin(), test_elements.end(), t) != test_elements.end(); }; + return make_lambda_xfunction(std::move(lambda), std::forward(element)); + } + + /** + * @ingroup logical_operators + * @brief isin + * + * Returns a boolean array of the same shape as ``element`` that is ``true`` where an element of + * ``element`` is in ``test_elements`` and ``False`` otherwise. + * @param element an \ref xexpression + * @param test_elements_begin iterator to the beginning of an array + * @param test_elements_end iterator to the end of an array + * @return a boolean array + */ + template ::value>> + inline auto isin(E&& element, I&& test_elements_begin, I&& test_elements_end) noexcept + { + auto lambda = [&test_elements_begin, &test_elements_end](const auto& t) { + return std::find(test_elements_begin, test_elements_end, t) != test_elements_end; }; + return make_lambda_xfunction(std::move(lambda), std::forward(element)); + } + + /** + * @ingroup logical_operators + * @brief in1d + * + * Returns a boolean array of the same shape as ``element`` that is ``true`` where an element of + * ``element`` is in ``test_elements`` and ``False`` otherwise. + * @param element an \ref xexpression + * @param test_elements an array + * @return a boolean array + */ + template + inline auto in1d(E&& element, std::initializer_list test_elements) noexcept + { + XTENSOR_ASSERT(element.dimension() == 1ul); + return isin(std::forward(element), std::forward>(test_elements)); + } + + /** + * @ingroup logical_operators + * @brief in1d + * + * Returns a boolean array of the same shape as ``element`` that is ``true`` where an element of + * ``element`` is in ``test_elements`` and ``False`` otherwise. + * @param element an \ref xexpression + * @param test_elements an array + * @return a boolean array + */ + template ::value>> + inline auto in1d(E&& element, F&& test_elements) noexcept + { + XTENSOR_ASSERT(element.dimension() == 1ul); + XTENSOR_ASSERT(test_elements.dimension() == 1ul); + return isin(std::forward(element), std::forward(test_elements)); + } + + /** + * @ingroup logical_operators + * @brief in1d + * + * Returns a boolean array of the same shape as ``element`` that is ``true`` where an element of + * ``element`` is in ``test_elements`` and ``False`` otherwise. + * @param element an \ref xexpression + * @param test_elements_begin iterator to the beginning of an array + * @param test_elements_end iterator to the end of an array + * @return a boolean array + */ + template ::value>> + inline auto in1d(E&& element, I&& test_elements_begin, I&& test_elements_end) noexcept + { + XTENSOR_ASSERT(element.dimension() == 1ul); + XTENSOR_ASSERT(test_elements.dimension() == 1ul); + return isin(std::forward(element), std::forward(test_elements_begin), std::forward(test_elements_end)); + } + +} + +#endif diff --git a/include/xtensor/xutils.hpp b/include/xtensor/xutils.hpp index 5a3f31bce..72b2066f3 100644 --- a/include/xtensor/xutils.hpp +++ b/include/xtensor/xutils.hpp @@ -609,6 +609,27 @@ namespace xt { }; + /****************************** + * is_iterator implementation * + ******************************/ + + template + struct is_iterator : std::false_type + { + }; + + template + struct is_iterator(), + std::declval() == std::declval(), + std::declval() != std::declval(), + ++ (*std::declval()), + (*std::declval()) ++, + std::true_type())>> + : std::true_type + { + }; + /******************************************** * xtrivial_default_construct implemenation * ********************************************/ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0bc704f33..2184a8a78 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -210,6 +210,7 @@ set(XTENSOR_TESTS test_xoptional.cpp test_xoptional_assembly_adaptor.cpp test_xoptional_assembly_storage.cpp + test_xset_operation.cpp test_xrandom.cpp test_xrepeat.cpp test_xsort.cpp diff --git a/test/test_xoperation.cpp b/test/test_xoperation.cpp index 4dccee0a4..f9748c097 100644 --- a/test/test_xoperation.cpp +++ b/test/test_xoperation.cpp @@ -839,4 +839,5 @@ namespace xt EXPECT_EQ(expected1, res3); EXPECT_EQ(expected2, res4); } + } diff --git a/test/test_xset_operation.cpp b/test/test_xset_operation.cpp new file mode 100644 index 000000000..58c8e34e3 --- /dev/null +++ b/test/test_xset_operation.cpp @@ -0,0 +1,39 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include "gtest/gtest.h" + +#include + +#include "xtensor/xarray.hpp" +#include "xtensor/xtensor.hpp" +#include "xtensor/xset_operation.hpp" + +namespace xt +{ + TEST(xset_operation, isin) + { + xt::xtensor a = {{1, 2, 1}, {0, 3, 1}}; + xt::xtensor b = {1, 2}; + xt::xtensor res = {{true, true, true}, {false, false, true}}; + EXPECT_EQ(xt::isin(a, b), res); + EXPECT_EQ(xt::isin(a, b.begin(), b.end()), res); + EXPECT_EQ(xt::isin(a, {1, 2}), res); + } + + TEST(xset_operation, in1d) + { + xt::xtensor a = {1, 2, 1, 0, 3, 5, 1}; + xt::xtensor b = {1, 2}; + xt::xtensor res = {true, true, true, false, false, false, true}; + EXPECT_EQ(xt::in1d(a, b), res); + EXPECT_EQ(xt::in1d(a, b.begin(), b.end()), res); + EXPECT_EQ(xt::in1d(a, {1, 2}), res); + } +} From 2ece5344cafd0d7b067936f6de7fe5e2a401c4b2 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Fri, 17 Apr 2020 15:42:43 +0200 Subject: [PATCH 019/606] Fixing capture of rvalue --- include/xtensor/xset_operation.hpp | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/include/xtensor/xset_operation.hpp b/include/xtensor/xset_operation.hpp index 9488054d8..9cb6b3fc2 100644 --- a/include/xtensor/xset_operation.hpp +++ b/include/xtensor/xset_operation.hpp @@ -26,6 +26,32 @@ namespace xt { + namespace detail + { + + template + struct make_lambda_isin_dispatch : std::integral_constant {}; + + template + struct make_lambda_isin_dispatch::value>> + : std::integral_constant {}; + + template + inline auto make_lambda_isin(E&& e, std::integral_constant) + { + return [e](const auto& t) { + return std::find(e.begin(), e.end(), t) != e.end(); }; + } + + template + inline auto make_lambda_isin(E&& e, std::integral_constant) + { + return [&e](const auto& t) { + return std::find(e.begin(), e.end(), t) != e.end(); }; + } + + } + /** * @ingroup logical_operators * @brief isin @@ -57,8 +83,7 @@ namespace xt template ::value>> inline auto isin(E&& element, F&& test_elements) noexcept { - auto lambda = [&test_elements](const auto& t) { - return std::find(test_elements.begin(), test_elements.end(), t) != test_elements.end(); }; + auto lambda = detail::make_lambda_isin(test_elements, detail::make_lambda_isin_dispatch{}); return make_lambda_xfunction(std::move(lambda), std::forward(element)); } From 6d00f5cee5f451a73abe44cf3d5b532f3abf5f92 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Fri, 17 Apr 2020 15:44:00 +0200 Subject: [PATCH 020/606] Fixing typo --- include/xtensor/xset_operation.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/xtensor/xset_operation.hpp b/include/xtensor/xset_operation.hpp index 9cb6b3fc2..422bd982b 100644 --- a/include/xtensor/xset_operation.hpp +++ b/include/xtensor/xset_operation.hpp @@ -156,7 +156,6 @@ namespace xt inline auto in1d(E&& element, I&& test_elements_begin, I&& test_elements_end) noexcept { XTENSOR_ASSERT(element.dimension() == 1ul); - XTENSOR_ASSERT(test_elements.dimension() == 1ul); return isin(std::forward(element), std::forward(test_elements_begin), std::forward(test_elements_end)); } From 87b7fe5f0d65980de74c3d16b65fcd93bd74272a Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Fri, 17 Apr 2020 16:32:33 +0200 Subject: [PATCH 021/606] Writing single include header --- CMakeLists.txt | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ffe5ee863..2680c9858 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,90 @@ endif() # Build # ===== +function(PREPEND var prefix) + set(listVar "") + foreach(f ${ARGN}) + list(APPEND listVar "${prefix}${f}") + endforeach(f) + set(${var} "${listVar}" PARENT_SCOPE) +endfunction() + +function(POSTFIX var postfix) + set(listVar "") + foreach(f ${ARGN}) + list(APPEND listVar "${f}${postfix}") + endforeach(f) + set(${var} "${listVar}" PARENT_SCOPE) +endfunction() + +set(XTENSOR_SINGLE_INCLUDE + xtensor/xaccessible.hpp + xtensor/xaccumulator.hpp + xtensor/xadapt.hpp + xtensor/xarray.hpp + xtensor/xassign.hpp + xtensor/xaxis_iterator.hpp + xtensor/xaxis_slice_iterator.hpp + xtensor/xbroadcast.hpp + xtensor/xbuffer_adaptor.hpp + xtensor/xbuilder.hpp + xtensor/xcomplex.hpp + xtensor/xcontainer.hpp + xtensor/xcsv.hpp + xtensor/xdynamic_view.hpp + xtensor/xeval.hpp + xtensor/xexception.hpp + xtensor/xexpression.hpp + # xtensor/xexpression_holder.hpp + xtensor/xexpression_traits.hpp + xtensor/xfixed.hpp + xtensor/xfunction.hpp + xtensor/xfunctor_view.hpp + xtensor/xgenerator.hpp + xtensor/xhistogram.hpp + xtensor/xindex_view.hpp + xtensor/xinfo.hpp + xtensor/xio.hpp + xtensor/xiterable.hpp + xtensor/xiterator.hpp + # xtensor/xjson.hpp + xtensor/xlayout.hpp + xtensor/xmanipulation.hpp + xtensor/xmasked_view.hpp + xtensor/xmath.hpp + # xtensor/xmime.hpp + xtensor/xnoalias.hpp + xtensor/xnorm.hpp + # xtensor/xnpy.hpp + xtensor/xoffset_view.hpp + xtensor/xoperation.hpp + xtensor/xoptional.hpp + xtensor/xoptional_assembly.hpp + xtensor/xoptional_assembly_base.hpp + xtensor/xoptional_assembly_storage.hpp + xtensor/xpad.hpp + xtensor/xrandom.hpp + xtensor/xreducer.hpp + xtensor/xrepeat.hpp + xtensor/xscalar.hpp + xtensor/xsemantic.hpp + xtensor/xshape.hpp + xtensor/xslice.hpp + xtensor/xsort.hpp + xtensor/xstorage.hpp + xtensor/xstrided_view.hpp + xtensor/xstrided_view_base.hpp + xtensor/xstrides.hpp + xtensor/xtensor.hpp + xtensor/xtensor_config.hpp + xtensor/xtensor_forward.hpp + xtensor/xtensor_simd.hpp + xtensor/xutils.hpp + xtensor/xvectorize.hpp + xtensor/xview.hpp + xtensor/xview_utils.hpp +) + set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xaccessible.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xaccumulator.hpp @@ -284,3 +368,16 @@ configure_file(${PROJECT_NAME}.pc.in @ONLY) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/") + +# Write single include +# ==================== + +PREPEND(XTENSOR_SINGLE_INCLUDE "#include <" ${XTENSOR_SINGLE_INCLUDE}) +POSTFIX(XTENSOR_SINGLE_INCLUDE ">" ${XTENSOR_SINGLE_INCLUDE}) +string(REPLACE ";" "\n" XTENSOR_SINGLE_INCLUDE "${XTENSOR_SINGLE_INCLUDE}") +string(CONCAT XTENSOR_SINGLE_INCLUDE "#ifndef XTENSOR\n" "#define XTENSOR\n\n" "${XTENSOR_SINGLE_INCLUDE}" "\n\n#endif\n") + +file(WRITE "${CMAKE_INSTALL_INCLUDEDIR}/xtensor.hpp" "${XTENSOR_SINGLE_INCLUDE}") + +install(FILES "${CMAKE_INSTALL_INCLUDEDIR}/xtensor.hpp" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) From 9a92d8db43bfcbeb7bbae79817f2869fde282c20 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Fri, 17 Apr 2020 17:44:39 +0200 Subject: [PATCH 022/606] Improving lvalue capture --- include/xtensor/xset_operation.hpp | 37 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/include/xtensor/xset_operation.hpp b/include/xtensor/xset_operation.hpp index 422bd982b..371b2b021 100644 --- a/include/xtensor/xset_operation.hpp +++ b/include/xtensor/xset_operation.hpp @@ -29,26 +29,25 @@ namespace xt namespace detail { - template - struct make_lambda_isin_dispatch : std::integral_constant {}; - - template - struct make_lambda_isin_dispatch::value>> - : std::integral_constant {}; - - template - inline auto make_lambda_isin(E&& e, std::integral_constant) + template + struct lambda_isin { - return [e](const auto& t) { - return std::find(e.begin(), e.end(), t) != e.end(); }; - } - - template - inline auto make_lambda_isin(E&& e, std::integral_constant) + template + static auto make(E&& e) + { + return [&e](const auto& t) { return std::find(e.begin(), e.end(), t) != e.end(); }; + } + }; + + template <> + struct lambda_isin { - return [&e](const auto& t) { - return std::find(e.begin(), e.end(), t) != e.end(); }; - } + template + static auto make(E&& e) + { + return [e](const auto& t) { return std::find(e.begin(), e.end(), t) != e.end(); }; + } + }; } @@ -83,7 +82,7 @@ namespace xt template ::value>> inline auto isin(E&& element, F&& test_elements) noexcept { - auto lambda = detail::make_lambda_isin(test_elements, detail::make_lambda_isin_dispatch{}); + auto lambda = detail::lambda_isin::value>::make(std::forward(test_elements)); return make_lambda_xfunction(std::move(lambda), std::forward(element)); } From d71ca44830b098348f61e59754abf3b9c8f85b62 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Fri, 17 Apr 2020 18:02:43 +0200 Subject: [PATCH 023/606] Updating PR template --- .github/PULL_REQUEST_TEMPLATE.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e2af2dd24..eed4e3b7c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,13 @@ -**Please check if your PR fulfills these requirements** +# Checklist -- The title and the commit message(s) are descriptive -- Small commits made to fix your PR have been squashed to avoid history pollution -- Tests have been added for new features or bug fixes -- API of new functions and classes are documented +- [ ] The title and commit message(s) are descriptive. +- [ ] Small commits made to fix your PR have been squashed to avoid history pollution. +- [ ] Tests have been added for new features or bug fixes. +- [ ] API of new functions and classes are documented. + +# Description + + From b88f67c55c69bbf9da47923f5648bafd0fe3be6c Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Sat, 18 Apr 2020 15:21:29 +0200 Subject: [PATCH 024/606] Simplifying excluding headers --- CMakeLists.txt | 76 ++++++-------------------------------------------- 1 file changed, 8 insertions(+), 68 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2680c9858..ec2e4b713 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,74 +128,6 @@ function(POSTFIX var postfix) set(${var} "${listVar}" PARENT_SCOPE) endfunction() -set(XTENSOR_SINGLE_INCLUDE - xtensor/xaccessible.hpp - xtensor/xaccumulator.hpp - xtensor/xadapt.hpp - xtensor/xarray.hpp - xtensor/xassign.hpp - xtensor/xaxis_iterator.hpp - xtensor/xaxis_slice_iterator.hpp - xtensor/xbroadcast.hpp - xtensor/xbuffer_adaptor.hpp - xtensor/xbuilder.hpp - xtensor/xcomplex.hpp - xtensor/xcontainer.hpp - xtensor/xcsv.hpp - xtensor/xdynamic_view.hpp - xtensor/xeval.hpp - xtensor/xexception.hpp - xtensor/xexpression.hpp - # xtensor/xexpression_holder.hpp - xtensor/xexpression_traits.hpp - xtensor/xfixed.hpp - xtensor/xfunction.hpp - xtensor/xfunctor_view.hpp - xtensor/xgenerator.hpp - xtensor/xhistogram.hpp - xtensor/xindex_view.hpp - xtensor/xinfo.hpp - xtensor/xio.hpp - xtensor/xiterable.hpp - xtensor/xiterator.hpp - # xtensor/xjson.hpp - xtensor/xlayout.hpp - xtensor/xmanipulation.hpp - xtensor/xmasked_view.hpp - xtensor/xmath.hpp - # xtensor/xmime.hpp - xtensor/xnoalias.hpp - xtensor/xnorm.hpp - # xtensor/xnpy.hpp - xtensor/xoffset_view.hpp - xtensor/xoperation.hpp - xtensor/xoptional.hpp - xtensor/xoptional_assembly.hpp - xtensor/xoptional_assembly_base.hpp - xtensor/xoptional_assembly_storage.hpp - xtensor/xpad.hpp - xtensor/xrandom.hpp - xtensor/xreducer.hpp - xtensor/xrepeat.hpp - xtensor/xscalar.hpp - xtensor/xsemantic.hpp - xtensor/xshape.hpp - xtensor/xslice.hpp - xtensor/xsort.hpp - xtensor/xstorage.hpp - xtensor/xstrided_view.hpp - xtensor/xstrided_view_base.hpp - xtensor/xstrides.hpp - xtensor/xtensor.hpp - xtensor/xtensor_config.hpp - xtensor/xtensor_forward.hpp - xtensor/xtensor_simd.hpp - xtensor/xutils.hpp - xtensor/xvectorize.hpp - xtensor/xview.hpp - xtensor/xview_utils.hpp -) - set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xaccessible.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xaccumulator.hpp @@ -372,6 +304,14 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc" # Write single include # ==================== +set(XTENSOR_SINGLE_INCLUDE ${XTENSOR_HEADERS}) +string(REPLACE "${XTENSOR_INCLUDE_DIR}/" "" XTENSOR_SINGLE_INCLUDE "${XTENSOR_SINGLE_INCLUDE}") +list(REMOVE_ITEM XTENSOR_SINGLE_INCLUDE + xtensor/xexpression_holder.hpp + xtensor/xjson.hpp + xtensor/xmime.hpp + xtensor/xnpy.hpp) + PREPEND(XTENSOR_SINGLE_INCLUDE "#include <" ${XTENSOR_SINGLE_INCLUDE}) POSTFIX(XTENSOR_SINGLE_INCLUDE ">" ${XTENSOR_SINGLE_INCLUDE}) string(REPLACE ";" "\n" XTENSOR_SINGLE_INCLUDE "${XTENSOR_SINGLE_INCLUDE}") From c06e594293d6b88b4b741aee99078f46839ea4d0 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Mon, 20 Apr 2020 14:32:34 +0200 Subject: [PATCH 025/606] Switching temporary directory of single-include header --- CMakeLists.txt | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec2e4b713..83b11fec9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,22 +112,6 @@ endif() # Build # ===== -function(PREPEND var prefix) - set(listVar "") - foreach(f ${ARGN}) - list(APPEND listVar "${prefix}${f}") - endforeach(f) - set(${var} "${listVar}" PARENT_SCOPE) -endfunction() - -function(POSTFIX var postfix) - set(listVar "") - foreach(f ${ARGN}) - list(APPEND listVar "${f}${postfix}") - endforeach(f) - set(${var} "${listVar}" PARENT_SCOPE) -endfunction() - set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xaccessible.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xaccumulator.hpp @@ -304,6 +288,22 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc" # Write single include # ==================== +function(PREPEND var prefix) + set(listVar "") + foreach(f ${ARGN}) + list(APPEND listVar "${prefix}${f}") + endforeach(f) + set(${var} "${listVar}" PARENT_SCOPE) +endfunction() + +function(POSTFIX var postfix) + set(listVar "") + foreach(f ${ARGN}) + list(APPEND listVar "${f}${postfix}") + endforeach(f) + set(${var} "${listVar}" PARENT_SCOPE) +endfunction() + set(XTENSOR_SINGLE_INCLUDE ${XTENSOR_HEADERS}) string(REPLACE "${XTENSOR_INCLUDE_DIR}/" "" XTENSOR_SINGLE_INCLUDE "${XTENSOR_SINGLE_INCLUDE}") list(REMOVE_ITEM XTENSOR_SINGLE_INCLUDE @@ -317,7 +317,7 @@ POSTFIX(XTENSOR_SINGLE_INCLUDE ">" ${XTENSOR_SINGLE_INCLUDE}) string(REPLACE ";" "\n" XTENSOR_SINGLE_INCLUDE "${XTENSOR_SINGLE_INCLUDE}") string(CONCAT XTENSOR_SINGLE_INCLUDE "#ifndef XTENSOR\n" "#define XTENSOR\n\n" "${XTENSOR_SINGLE_INCLUDE}" "\n\n#endif\n") -file(WRITE "${CMAKE_INSTALL_INCLUDEDIR}/xtensor.hpp" "${XTENSOR_SINGLE_INCLUDE}") +file(WRITE "${CMAKE_BINARY_DIR}/xtensor.hpp" "${XTENSOR_SINGLE_INCLUDE}") -install(FILES "${CMAKE_INSTALL_INCLUDEDIR}/xtensor.hpp" +install(FILES "${CMAKE_BINARY_DIR}/xtensor.hpp" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) From f2a00396ff26b5cf2d1429310673b5d3d0e0d57e Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Sun, 3 May 2020 16:22:04 +0200 Subject: [PATCH 026/606] Adding details for xt::random to docs --- docs/source/api/xrandom.rst | 20 ++++++ docs/source/histogram.rst | 2 + docs/source/index.rst | 3 +- docs/source/numpy.rst | 4 ++ docs/source/random.rst | 135 ++++++++++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 docs/source/random.rst diff --git a/docs/source/api/xrandom.rst b/docs/source/api/xrandom.rst index 9db89b9aa..fd3bfc48e 100644 --- a/docs/source/api/xrandom.rst +++ b/docs/source/api/xrandom.rst @@ -11,62 +11,82 @@ Defined in ``xtensor/xrandom.hpp`` .. warning:: xtensor uses a lazy generator for random numbers. You need to assign them or use ``eval`` to keep the generated values consistent. +.. _random-get_default_random_engine-function-reference: .. doxygenfunction:: xt::random::get_default_random_engine :project: xtensor +.. _random-seed-function-reference: .. doxygenfunction:: xt::random::seed :project: xtensor +.. _random-rand-function-reference: .. doxygenfunction:: xt::random::rand(const S&, T, T, E&) :project: xtensor +.. _random-randint-function-reference: .. doxygenfunction:: xt::random::randint(const S&, T, T, E&) :project: xtensor +.. _random-randn-function-reference: .. doxygenfunction:: xt::random::randn(const S&, T, T, E&) :project: xtensor +.. _random-binomial-function-reference: .. doxygenfunction:: xt::random::binomial(const S&, T, D, E&) :project: xtensor +.. _random-geometric-function-reference: .. doxygenfunction:: xt::random::geometric(const S&, D, E&) :project: xtensor +.. _random-negative_binomial-function-reference: .. doxygenfunction:: xt::random::negative_binomial(const S&, T, D, E&) :project: xtensor +.. _random-poisson-function-reference: .. doxygenfunction:: xt::random::poisson(const S&, D, E&) :project: xtensor +.. _random-exponential-function-reference: .. doxygenfunction:: xt::random::exponential(const S&, T, E&) :project: xtensor +.. _random-gamma-function-reference: .. doxygenfunction:: xt::random::gamma(const S&, T, T, E&) :project: xtensor +.. _random-weibull-function-reference: .. doxygenfunction:: xt::random::weibull(const S&, T, T, E&) :project: xtensor +.. _random-extreme_value-function-reference: .. doxygenfunction:: xt::random::extreme_value(const S&, T, T, E&) :project: xtensor +.. _random-lognormal-function-reference: .. doxygenfunction:: xt::random::lognormal(const S&, T, T, E&) :project: xtensor +.. _random-cauchy-function-reference: .. doxygenfunction:: xt::random::cauchy(const S&, T, T, E&) :project: xtensor +.. _random-fisher_f-function-reference: .. doxygenfunction:: xt::random::fisher_f(const S&, T, T, E&) :project: xtensor +.. _random-student_t-function-reference: .. doxygenfunction:: xt::random::student_t(const S&, T, E&) :project: xtensor +.. _random-choice-function-reference: .. doxygenfunction:: xt::random::choice :project: xtensor +.. _random-shuffle-function-reference: .. doxygenfunction:: xt::random::shuffle :project: xtensor +.. _random-permutation-function-reference: .. doxygenfunction:: xt::random::permutation(T, E&) :project: xtensor diff --git a/docs/source/histogram.rst b/docs/source/histogram.rst index 4cf471d3d..f42b6a7d5 100644 --- a/docs/source/histogram.rst +++ b/docs/source/histogram.rst @@ -4,6 +4,8 @@ The full license is in the file LICENSE, distributed with this software. +.. _histogram: + Histogram ========= diff --git a/docs/source/index.rst b/docs/source/index.rst index fb22231e6..2a9d4b34d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -63,11 +63,12 @@ for details. scalar adaptor operator - histogram view indices builder missing + histogram + random file_loading build-options pitfall diff --git a/docs/source/numpy.rst b/docs/source/numpy.rst index 36d1cc723..63569441f 100644 --- a/docs/source/numpy.rst +++ b/docs/source/numpy.rst @@ -189,6 +189,8 @@ The random module provides simple ways to create random tensor expressions, lazi | ``np.random.permutation(30)`` | ``xt::random::permutation(30)`` | +-----------------------------------------------+-----------------------------------------------+ +See :ref:`random`. + Concatenation, splitting, squeezing ----------------------------------- @@ -610,6 +612,8 @@ xtensor universal functions are provided for a large set number of mathematical | ``np.bincount(arr)`` | ``xt::bincount(arr)`` | +-------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ +See :ref:`histogram`. + Linear algebra -------------- diff --git a/docs/source/random.rst b/docs/source/random.rst new file mode 100644 index 000000000..9ed21c010 --- /dev/null +++ b/docs/source/random.rst @@ -0,0 +1,135 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +.. _random: + +****** +Random +****** + +xt::random::seed +================ + +:ref:`xt::random::seed ` + +Set seed for random number generator. A common practice to get a 'real' random number is to use: + +.. code-block:: cpp + + #include + + ... + + xt::random::seed(time(NULL)); + +xt::random::rand +================ + +:ref:`xt::random::rand ` + +xt::random::randint +=================== + +:ref:`xt::random::randint ` + +xt::random::randn +================= + +:ref:`xt::random::randn ` + +xt::random::binomial +==================== + +:ref:`xt::random::binomial ` + +xt::random::geometric +===================== + +:ref:`xt::random::geometric ` + +xt::random::negative_binomial +============================= + +:ref:`xt::random::negative_binomial ` + +xt::random::poisson +=================== + +:ref:`xt::random::poisson ` + +xt::random::exponential +======================= + +:ref:`xt::random::exponential ` + +xt::random::gamma +================= + +:ref:`xt::random::gamma ` + +Produces (an array of) random positive floating-point values, distributed according to the probability density: + +.. math:: + + P(x) = x^{\alpha-1} \frac{e^{-x / \beta}}{\beta^\alpha \; \Gamma(\alpha)} + +where :math:`\alpha` is the shape (also known as :math:`k`) and :math:`\beta` the scale (also known as :math:`\theta`), and :math:`\Gamma` is the Gamma function. + +.. note:: + + Do not confuse the first argument of ``xt::random``, the shape of the output array, with the parameter :math:`alpha`. + +.. seealso:: + + * `numpy.random.gamma `_ + * `std::gamma_distribution `_ + * `Weisstein, Eric W. "Gamma Distribution." From MathWorld – A Wolfram Web Resource. `_ + * `Wikipedia, "Gamma distribution". `_ + +xt::random::weibull +=================== + +:ref:`xt::random::weibull ` + +xt::random::extreme_value +========================= + +:ref:`xt::random::extreme_value ` + +xt::random::lognormal +===================== + +:ref:`xt::random::lognormal ` + +xt::random::cauchy +================== + +:ref:`xt::random::cauchy ` + +xt::random::fisher_f +==================== + +:ref:`xt::random::fisher_f ` + +xt::random::student_t +===================== + +:ref:`xt::random::student_t ` + +xt::random::choice +================== + +:ref:`xt::random::choice ` + +xt::random::shuffle +=================== + +:ref:`xt::random::shuffle ` + +xt::random::permutation +======================= + +:ref:`xt::random::permutation ` From 9c8aa63cb950555d0c3136c49a470b7adf81dc06 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Mon, 4 May 2020 11:21:38 +0200 Subject: [PATCH 027/606] Fixing compiler warnings --- include/xtensor/xfunction.hpp | 5 ++++- include/xtensor/xmath.hpp | 10 ++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/xtensor/xfunction.hpp b/include/xtensor/xfunction.hpp index 702ead106..b7f8fab78 100644 --- a/include/xtensor/xfunction.hpp +++ b/include/xtensor/xfunction.hpp @@ -50,7 +50,10 @@ namespace xt bool is_trivial; bool is_initialized; - xfunction_cache_impl() : shape(xtl::make_sequence(0, std::size_t(0))), is_trivial(false), is_initialized(false) {} + xfunction_cache_impl() : + shape(xtl::make_sequence(0, std::size_t(0))), + is_initialized(false), + is_trivial(false) {} }; template diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index 13e1bad27..ac6e3b68b 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -1925,7 +1925,6 @@ namespace detail { inline auto mean(E&& e, const I (&axes)[N], const D& ddof, EVS es) { auto s = sum(std::forward(e), axes, es); - using size_type = typename std::decay_t::size_type; return detail::mean_division(std::move(s), e.size() - ddof); } #endif @@ -1936,7 +1935,6 @@ namespace detail { { using value_type = typename std::conditional_t::value, double, T>; auto size = e.size(); - using size_type = decltype(size); return sum(std::forward(e), es) / static_cast(size - ddof); } } @@ -3033,7 +3031,7 @@ namespace detail { /** * @brief Returns the covariance matrix - * + * * @param x one or two dimensional array * @param y optional one-dimensional array to build covariance to x */ @@ -3053,7 +3051,7 @@ namespace detail { covar(0, 0) = std::inner_product(x_norm.begin(), x_norm.end(), x_norm.begin(), 0.0) / value_type(s[0] - 1); return covar; } - + XTENSOR_ASSERT( x.dimension() == 2 ); auto covar = eval(zeros({ s[0], s[0] })); @@ -3065,12 +3063,12 @@ namespace detail { auto xi = strided_view(x_norm, { range(i, i + 1), all() }); for (size_type j = i; j < s[0]; j++) { - auto xj = strided_view(x_norm, { range(j, j + 1), all() }); + auto xj = strided_view(x_norm, { range(j, j + 1), all() }); covar(j, i) = std::inner_product(xi.begin(), xi.end(), xj.begin(), 0.0) / value_type(s[1] - 1); } } return eval(covar + transpose(covar) - diag(diagonal(covar))); - } + } else { return cov(eval(stack(xtuple(x, y)))); From 7b1f7dd4a1a26371ff0084a1e3f63f757bcd680e Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Tue, 5 May 2020 21:54:18 +0200 Subject: [PATCH 028/606] Adding "digitize", "searchsorted", and "bin_items" (#2037) Adding "digitize", "searchsorted", and "bin_items" --- docs/source/api/function_index.rst | 1 + docs/source/api/xhistogram.rst | 9 +++ docs/source/api/xset_operation.rst | 34 +++++++++ docs/source/numpy.rst | 4 ++ include/xtensor/xhistogram.hpp | 110 +++++++++++++++++++++++------ include/xtensor/xset_operation.hpp | 35 +++++++++ test/test_xhistogram.cpp | 42 ++++++++--- test/test_xset_operation.cpp | 11 +++ 8 files changed, 216 insertions(+), 30 deletions(-) create mode 100644 docs/source/api/xset_operation.rst diff --git a/docs/source/api/function_index.rst b/docs/source/api/function_index.rst index fb1a31319..b4e3a22cf 100644 --- a/docs/source/api/function_index.rst +++ b/docs/source/api/function_index.rst @@ -16,6 +16,7 @@ Functions and generators xbuilder xmanipulation xsort + xset_operation xrandom xhistogram xpad diff --git a/docs/source/api/xhistogram.rst b/docs/source/api/xhistogram.rst index a268e3141..83bdbbc58 100644 --- a/docs/source/api/xhistogram.rst +++ b/docs/source/api/xhistogram.rst @@ -21,6 +21,12 @@ Defined in ``xtensor/xhistogram.hpp`` .. doxygenfunction:: xt::histogram_bin_edges(E1&&, E2&&, E3, E3, std::size_t, histogram_algorithm) :project: xtensor +.. doxygenfunction:: xt::digitize(E1&&, E2&&, E3&&, bool, bool) + :project: xtensor + +.. doxygenfunction:: xt::bin_items(size_t, E&&) + :project: xtensor + Further overloads ----------------- @@ -41,3 +47,6 @@ Further overloads .. doxygenfunction:: xt::histogram_bin_edges(E1&&, std::size_t, histogram_algorithm) :project: xtensor + +.. doxygenfunction:: xt::bin_items(size_t, size_t) + :project: xtensor diff --git a/docs/source/api/xset_operation.rst b/docs/source/api/xset_operation.rst new file mode 100644 index 000000000..2787ad200 --- /dev/null +++ b/docs/source/api/xset_operation.rst @@ -0,0 +1,34 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +xset_operation +============== + +Defined in ``xtensor/xset_operation.hpp`` + +.. doxygenenum:: xt::isin(E&&, F&&) + :project: xtensor + +.. doxygenenum:: xt::in1d(E&&, F&&) + :project: xtensor + +.. doxygenenum:: xt::searchsorted(E1&&, E2&&, bool) + :project: xtensor + +Further overloads +----------------- + +.. doxygenenum:: xt::isin(E&&, std::initializer_list) + :project: xtensor + +.. doxygenenum:: xt::isin(E&&, I&&, I&&) + :project: xtensor + +.. doxygenenum:: xt::in1d(E&&, std::initializer_list) + :project: xtensor + +.. doxygenenum:: xt::in1d(E&&, I&&, I&&) + :project: xtensor diff --git a/docs/source/numpy.rst b/docs/source/numpy.rst index 63569441f..d57391054 100644 --- a/docs/source/numpy.rst +++ b/docs/source/numpy.rst @@ -599,6 +599,8 @@ xtensor universal functions are provided for a large set number of mathematical +-----------------------------------------------+-----------------------------------------------+ | ``np.isfinite(a)`` | ``xt::isfinite(a)`` | +-----------------------------------------------+-----------------------------------------------+ +| ``np.searchsorted(a, v[, side])`` | ``xt::searchsorted(a, v[, right])`` | ++-----------------------------------------------+-----------------------------------------------+ **Histogram:** @@ -611,6 +613,8 @@ xtensor universal functions are provided for a large set number of mathematical +-------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ | ``np.bincount(arr)`` | ``xt::bincount(arr)`` | +-------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ +| ``np.digitize(data, bin_edges[, right])`` | ``xt::digitize(data, bin_edges[, right][, assume_sorted])`` | ++-------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ See :ref:`histogram`. diff --git a/include/xtensor/xhistogram.hpp b/include/xtensor/xhistogram.hpp index 50a05f67d..0228d26d2 100644 --- a/include/xtensor/xhistogram.hpp +++ b/include/xtensor/xhistogram.hpp @@ -16,15 +16,40 @@ #include "xtensor.hpp" #include "xsort.hpp" +#include "xset_operation.hpp" +#include "xview.hpp" + +using namespace xt::placeholders; namespace xt { + /** + * @ingroup digitize + * @brief Return the indices of the bins to which each value in input array belongs. + * + * @param data The data. + * @param bin_edges The bin-edges. It has to be 1-dimensional and monotonic. + * @param right Indicating whether the intervals include the right or the left bin edge. + * @return Output array of indices, of same shape as x. + */ + template + inline auto digitize(E1&& data, E2&& bin_edges, bool right = false) + { + XTENSOR_ASSERT(bin_edges.dimension() == 1); + XTENSOR_ASSERT(bin_edges.size() >= 2); + XTENSOR_ASSERT(std::is_sorted(bin_edges.cbegin(), bin_edges.cend())); + XTENSOR_ASSERT(xt::amin(data)[0] >= bin_edges[0]); + XTENSOR_ASSERT(xt::amax(data)[0] <= bin_edges[bin_edges.size() - 1]); + + return xt::searchsorted(std::forward(bin_edges), std::forward(data), right); + } + /** * @ingroup histogram * @brief Compute the histogram of a set of data. * * @param data The data. - * @param bin_edges The bin-edges. + * @param bin_edges The bin-edges. It has to be 1-dimensional and monotonic. * @param weights Weight factors corresponding to each data-point. * @param density If true the resulting integral is normalized to 1. [default: false] * @return An one-dimensional xarray, length: bin_edges.size()-1. @@ -32,54 +57,38 @@ namespace xt template inline auto histogram(E1&& data, E2&& bin_edges, E3&& weights, bool density = false) { - // alias counter and value type using size_type = common_size_type_t, std::decay_t, std::decay_t>; using value_type = typename std::decay_t::value_type; - // basic checks - // - rank XTENSOR_ASSERT(data.dimension() == 1); XTENSOR_ASSERT(weights.dimension() == 1); XTENSOR_ASSERT(bin_edges.dimension() == 1); - // - size XTENSOR_ASSERT(weights.size() == data.size()); XTENSOR_ASSERT(bin_edges.size() >= 2); - // - bin-edges must be sorted XTENSOR_ASSERT(std::is_sorted(bin_edges.cbegin(), bin_edges.cend())); - // - data must be enclosed in the bins XTENSOR_ASSERT(xt::amin(data)[0] >= bin_edges[0]); XTENSOR_ASSERT(xt::amax(data)[0] <= bin_edges[bin_edges.size() - 1]); - // initialize output xt::xtensor count = xt::zeros({ bin_edges.size() - 1 }); - // indices that sort "data" - auto isort = xt::argsort(data); + auto sorter = xt::argsort(data); - // index of the current bin size_type ibin = 0; - // fill the histogram: loop over (sorted) data - for (auto& idx : isort) + for (auto& idx : sorter) { - // - proceed to the relevant bin while (data[idx] >= bin_edges[ibin + 1] && ibin < bin_edges.size() - 2) { ++ibin; } - // - update the count count[ibin] += weights[idx]; } - // cast type xt::xtensor prob = xt::cast(count); - // normalize if (density) { - // - size as doubles R n = static_cast(data.size()); - // - apply normalization for (size_type i = 0; i < bin_edges.size() - 1; ++i) { prob[i] /= (static_cast(bin_edges[i + 1] - bin_edges[i]) * n); @@ -244,7 +253,7 @@ namespace xt case histogram_algorithm::uniform: { // indices that sort "data" - auto isort = xt::argsort(data); + auto sorter = xt::argsort(data); // histogram: all of equal 'height' // - height @@ -272,13 +281,15 @@ namespace xt { if (cum_weight >= count[ibin]) { - bin_edges[ibin + 1] = data[isort[i]]; + bin_edges[ibin + 1] = data[sorter[i]]; ++ibin; } - cum_weight += weights[isort[i]]; + cum_weight += weights[sorter[i]]; } return bin_edges; } + + // bins of equal width default: { xt::xtensor bin_edges @@ -426,6 +437,61 @@ namespace xt xt::ones::value_type>(data.shape()), minlength); } + + /** + * Get the number of items in each bin, given the fraction of items per bin. + * The output is such that the total number of items of all bins is exactly "N". + * + * @param N the number of items to distribute + * @param weights fraction of items per bin: a 1D container whose size is the number of bins + * + * @return 1D container with the number of items per bin + */ + template + inline xt::xtensor bin_items(size_t N, E&& weights) + { + if (weights.size() <= std::size_t(1)) + { + xt::xtensor n = N * xt::ones({1}); + return n; + } + + using value_type = typename std::decay_t::value_type; + + XTENSOR_ASSERT(xt::all(weights >= static_cast(0))); + XTENSOR_ASSERT(xt::sum(weights)() > static_cast(0)); + + xt::xtensor P = xt::cast(weights) / static_cast(xt::sum(weights)()); + xt::xtensor n = xt::ceil(static_cast(N) * P); + + if (xt::sum(n)() == N) + { + return n; + } + + xt::xtensor d = xt::zeros(P.shape()); + xt::xtensor sorter = xt::argsort(P); + sorter = xt::view(sorter, xt::range(P.size(), _, -1)); + sorter = xt::view(sorter, xt::range(0, xt::sum(n)(0) - N)); + xt::view(d, xt::keep(sorter)) = 1; + n -= d; + + return n; + } + + /** + * Get the number of items in each bin, with each bin having approximately the same number of + * items in it,under the constraint that the total number of items of all bins is exactly "N". + * + * @param N the number of items to distribute + * @param bins the number of bins + * + * @return 1D container with the number of items per bin + */ + inline xt::xtensor bin_items(size_t N, size_t bins) + { + return bin_items(N, xt::ones({bins})); + } } #endif diff --git a/include/xtensor/xset_operation.hpp b/include/xtensor/xset_operation.hpp index 371b2b021..3addadb5b 100644 --- a/include/xtensor/xset_operation.hpp +++ b/include/xtensor/xset_operation.hpp @@ -158,6 +158,41 @@ namespace xt return isin(std::forward(element), std::forward(test_elements_begin), std::forward(test_elements_end)); } + /** + * @ingroup searchsorted + * @brief Find indices where elements should be inserted to maintain order. + * + * @param a Input array: sorted (array_like). + * @param v Values to insert into a (array_like). + * @param right If ``false``, the index of the first suitable location found is given. + * @return Array of insertion points with the same shape as v. + */ + template + inline auto searchsorted(E1&& a, E2&& v, bool right = true) + { + XTENSOR_ASSERT(std::is_sorted(a.cbegin(), a.cend())); + + auto out = xt::empty(v.shape()); + + if (right) + { + for (size_t i = 0; i < v.size(); ++i) + { + out(i) = std::lower_bound(a.cbegin(), a.cend(), v(i)) - a.cbegin(); + } + } + else + { + for (size_t i = 0; i < v.size(); ++i) + { + out(i) = std::upper_bound(a.cbegin(), a.cend(), v(i)) - a.cbegin(); + } + } + + + return out; + } + } #endif diff --git a/test/test_xhistogram.cpp b/test/test_xhistogram.cpp index 434b46eaf..79c7aeb26 100644 --- a/test/test_xhistogram.cpp +++ b/test/test_xhistogram.cpp @@ -19,30 +19,30 @@ namespace xt { TEST(xhistogram, histogram) { - xt::xtensor data = {1., 1., 2., 2.}; + xt::xtensor data = {1., 1., 2., 2.}; { - xt::xtensor count = xt::histogram(data, std::size_t(2)); + xt::xtensor count = xt::histogram(data, std::size_t(2)); EXPECT_EQ(count.size(), std::size_t(2) ); - EXPECT_EQ(count[0] , 2.); - EXPECT_EQ(count[1] , 2.); + EXPECT_EQ(count(0), 2.); + EXPECT_EQ(count(1), 2.); } { - xt::xtensor count = xt::histogram(data, + xt::xtensor count = xt::histogram(data, xt::histogram_bin_edges(data, std::size_t(2), xt::histogram_algorithm::uniform) ); EXPECT_EQ(count.size(), std::size_t(2)); - EXPECT_EQ(count[0] , 2.); - EXPECT_EQ(count[1] , 2.); + EXPECT_EQ(count(0), 2.); + EXPECT_EQ(count(1), 2.); } } TEST(xhistogram, bincount) { - xtensor data = {1,2,3,1,1,1,1,2,3,2,3,3,3,3}; + xtensor data = {1, 2, 3, 1, 1, 1, 1, 2, 3, 2, 3, 3, 3, 3}; xtensor weights = xt::ones(data.shape()) * 3; xtensor expc = {0, 5, 3, 6}; auto bc = bincount(data); @@ -55,4 +55,30 @@ namespace xt EXPECT_EQ(bc3.size(), std::size_t(10)); EXPECT_EQ(bc3(3), expc(3)); } + + TEST(xhistogram, digitize) + { + xt::xtensor bin_edges = {0, 10, 20, 30}; + xt::xtensor data = {1, 12, 2, 21, 11, 10}; + xt::xtensor res_left = {1, 2, 1, 3, 2, 2}; + xt::xtensor res_right = {1, 2, 1, 3, 2, 1}; + EXPECT_EQ(xt::digitize(data, bin_edges), res_left); + EXPECT_EQ(xt::digitize(data, bin_edges, false), res_left); + EXPECT_EQ(xt::digitize(data, bin_edges, true), res_right); + } + + + TEST(xhistogram, bin_items_1) + { + xt::xtensor a = xt::bin_items(11, xt::xtensor{0.9, 0.0, 0.0, 0.1}); + xt::xtensor b = {9, 0, 0, 2}; + EXPECT_EQ(a, b); + } + + TEST(xhistogram, bin_items_2) + { + xt::xtensor a = xt::bin_items(11, xt::xtensor{0.25, 0.25, 0.25, 0.25}); + xt::xtensor b = {3, 3, 3, 2}; + EXPECT_EQ(a, b); + } } diff --git a/test/test_xset_operation.cpp b/test/test_xset_operation.cpp index 58c8e34e3..fa18cf4e8 100644 --- a/test/test_xset_operation.cpp +++ b/test/test_xset_operation.cpp @@ -36,4 +36,15 @@ namespace xt EXPECT_EQ(xt::in1d(a, b.begin(), b.end()), res); EXPECT_EQ(xt::in1d(a, {1, 2}), res); } + + TEST(xset_operation, searchsorted) + { + xt::xtensor a = {1, 2, 7, 8, 20}; + xt::xtensor v = {9, 2, 2, 3, 22, 0}; + xt::xtensor res_right = {4, 1, 1, 2, 5, 0}; + xt::xtensor res_left = {4, 2, 2, 2, 5, 0}; + EXPECT_EQ(xt::searchsorted(a, v), res_right); + EXPECT_EQ(xt::searchsorted(a, v, true), res_right); + EXPECT_EQ(xt::searchsorted(a, v, false), res_left); + } } From 996d83837cf9a1509203153b84c4d3b964c654e5 Mon Sep 17 00:00:00 2001 From: Zaripov Kamil Date: Thu, 14 May 2020 01:57:22 +0300 Subject: [PATCH 029/606] Fixed error with zero tensor size in xt::mean --- include/xtensor/xmath.hpp | 11 +++++++---- test/test_xreducer.cpp | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index ac6e3b68b..126817b31 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -1909,23 +1909,26 @@ namespace detail { { // sum cannot always be a double. It could be a complex number which cannot operate on // std::plus. + const auto size = e.size(); auto s = sum(std::forward(e), std::forward(axes), es); - return mean_division(std::move(s), e.size() - ddof); + return mean_division(std::move(s), size - ddof); } #ifdef X_OLD_CLANG template inline auto mean(E&& e, std::initializer_list axes, const D& ddof, EVS es) { + const auto size = e.size(); auto s = sum(std::forward(e), axes, es); - return detail::mean_division(std::move(s), e.size() - ddof); + return detail::mean_division(std::move(s), size - ddof); } #else template inline auto mean(E&& e, const I (&axes)[N], const D& ddof, EVS es) { + const auto size = e.size(); auto s = sum(std::forward(e), axes, es); - return detail::mean_division(std::move(s), e.size() - ddof); + return detail::mean_division(std::move(s), size - ddof); } #endif @@ -1934,7 +1937,7 @@ namespace detail { inline auto mean_noaxis(E&& e, const D& ddof, EVS es) { using value_type = typename std::conditional_t::value, double, T>; - auto size = e.size(); + const auto size = e.size(); return sum(std::forward(e), es) / static_cast(size - ddof); } } diff --git a/test/test_xreducer.cpp b/test/test_xreducer.cpp index 81f94cccc..0e25f2197 100644 --- a/test/test_xreducer.cpp +++ b/test/test_xreducer.cpp @@ -243,6 +243,9 @@ namespace xt xarray c = {1, 2}; EXPECT_EQ(mean(c)(), 1.5); + + const auto rvalue_xarray = [] () { return xtensor({1, 2}); }; + EXPECT_EQ(mean(rvalue_xarray(), {0})(), 1.5); } TEST(xreducer, average) From 44ec96add6df3a1746ddabe1ee2bb7b96a70a1bf Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 20 May 2020 18:55:37 +0200 Subject: [PATCH 030/606] Fixed build with clang-cl --- .azure-pipelines/azure-pipelines-win.yml | 2 +- .gitignore | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 77ea6102f..4598db9ad 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -34,7 +34,7 @@ jobs: # Create conda enviroment # Note: conda activate doesn't work here, because it creates a new shell! - script: | - conda install cmake ^ + conda install cmake==3.14.0 ^ gtest==1.10.0 ^ ninja ^ nlohmann_json ^ diff --git a/.gitignore b/.gitignore index 2be69c28d..fa2b2e104 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ *.swp *~ +# Generated directory +include/tmp/ + # Build directory build/ From a21a1de3ce233344faf5870092341d2677cd2974 Mon Sep 17 00:00:00 2001 From: Phidias Chiang Date: Fri, 22 May 2020 11:28:15 +0800 Subject: [PATCH 031/606] Fix initialize order Signed-off-by: Phidias Chiang --- include/xtensor/xfunction.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xtensor/xfunction.hpp b/include/xtensor/xfunction.hpp index b7f8fab78..5f1c4d7e0 100644 --- a/include/xtensor/xfunction.hpp +++ b/include/xtensor/xfunction.hpp @@ -52,8 +52,8 @@ namespace xt xfunction_cache_impl() : shape(xtl::make_sequence(0, std::size_t(0))), - is_initialized(false), - is_trivial(false) {} + is_trivial(false), + is_initialized(false) {} }; template From e339c576f2e852c3d48afd1a6842c9c6e5dd01e7 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 26 May 2020 11:43:54 +0200 Subject: [PATCH 032/606] adapt_smart_ptr overlads accepting STL-like container as shape --- docs/source/api/xarray_adaptor.rst | 4 +- docs/source/api/xtensor_adaptor.rst | 6 ++ include/xtensor/xadapt.hpp | 126 +++++++++++++++++++++++++--- test/test_xadapt.cpp | 36 ++++++++ 4 files changed, 160 insertions(+), 12 deletions(-) diff --git a/docs/source/api/xarray_adaptor.rst b/docs/source/api/xarray_adaptor.rst index 48e77b252..6764a445b 100644 --- a/docs/source/api/xarray_adaptor.rst +++ b/docs/source/api/xarray_adaptor.rst @@ -36,8 +36,8 @@ Defined in ``xtensor/xadapt.hpp`` .. doxygenfunction:: xt::adapt(T (&)[N], SC&&, SS&&) :project: xtensor -.. doxygenfunction:: xt::adapt_smart_ptr(P&&, const I (&)[N]) +.. doxygenfunction:: xt::adapt_smart_ptr(P&&, const SC&, layout_type) :project: xtensor -.. doxygenfunction:: xt::adapt_smart_ptr(P&&, const I (&)[N], D&&) +.. doxygenfunction:: xt::adapt_smart_ptr(P&&, const SC&, D&&, layout_type) :project: xtensor diff --git a/docs/source/api/xtensor_adaptor.rst b/docs/source/api/xtensor_adaptor.rst index da783c42d..8d02fd424 100644 --- a/docs/source/api/xtensor_adaptor.rst +++ b/docs/source/api/xtensor_adaptor.rst @@ -41,3 +41,9 @@ Defined in ``xtensor/xadapt.hpp`` .. doxygenfunction:: xt::adapt(T (&)[N], SC&&, SS&&) :project: xtensor + +.. doxygenfunction:: xt::adapt_smart_ptr(P&&, const I (&)[N], layout_type) + :project: xtensor + +.. doxygenfunction:: xt::adapt_smart_ptr(P&&, const I (&)[N], D&&, layout_type) + :project: xtensor diff --git a/include/xtensor/xadapt.hpp b/include/xtensor/xadapt.hpp index 659114124..21cc7e5fb 100644 --- a/include/xtensor/xadapt.hpp +++ b/include/xtensor/xadapt.hpp @@ -412,6 +412,108 @@ namespace xt } #endif + /***************************** + * smart_ptr adapter builder * + *****************************/ + + /** + * Adapt a smart pointer to a typed memory block (unique_ptr or shared_ptr) + * + * \code{.cpp} + * #include + * #include + * + * std::shared_ptr sptr(new double[8], std::default_delete()); + * sptr.get()[2] = 321.; + * std::vector shape = {4, 2}; + * auto xptr = adapt_smart_ptr(sptr, shape); + * xptr(1, 3) = 123.; + * std::cout << xptr; + * \endcode + * + * @param smart_ptr a smart pointer to a memory block of T[] + * @param shape The desired shape + * @param l The desired memory layout + * + * @return xarray_adaptor for memory + */ + template >)> + auto adapt_smart_ptr(P&& smart_ptr, const SC& shape, layout_type l = L) + { + using buffer_adaptor = xbuffer_adaptor>; + return xarray_adaptor>( + buffer_adaptor(smart_ptr.get(), compute_size(shape), std::forward

(smart_ptr)), + shape, + l + ); + + } + + /** + * Adapt a smart pointer (shared_ptr or unique_ptr) + * + * This function allows to automatically adapt a shared or unique pointer to + * a given shape and operate naturally on it. Memory will be automatically + * handled by the smart pointer implementation. + * + * \code{.cpp} + * #include + * #include + * + * struct Buffer { + * Buffer(std::vector& buf) : m_buf(buf) {} + * ~Buffer() { std::cout << "deleted" << std::endl; } + * std::vector m_buf; + * }; + * + * auto data = std::vector{1,2,3,4,5,6,7,8}; + * auto shared_buf = std::make_shared(data); + * auto unique_buf = std::make_unique(data); + * + * std::cout << shared_buf.use_count() << std::endl; + * { + * std::vector shape = {2, 4}; + * auto obj = adapt_smart_ptr(shared_buf.get()->m_buf.data(), + * shape, shared_buf); + * // Use count increased to 2 + * std::cout << shared_buf.use_count() << std::endl; + * std::cout << obj << std::endl; + * } + * // Use count reset to 1 + * std::cout << shared_buf.use_count() << std::endl; + * + * { + * std::vector shape = {2, 4}; + * auto obj = adapt_smart_ptr(unique_buf.get()->m_buf.data(), + * shape, std::move(unique_buf)); + * std::cout << obj << std::endl; + * } + * \endcode + * + * @param data_ptr A pointer to a typed data block (e.g. double*) + * @param shape The desired shape + * @param smart_ptr A smart pointer to move or copy, in order to manage memory + * @param l The desired memory layout + * + * @return xarray_adaptor on the memory + */ + template >, + detail::not_a_layout>)> + auto adapt_smart_ptr(P&& data_ptr, const SC& shape, D&& smart_ptr, layout_type l = L) + { + using buffer_adaptor = xbuffer_adaptor>; + + return xarray_adaptor>( + buffer_adaptor(data_ptr, compute_size(shape), std::forward(smart_ptr)), + shape, + l + ); + } + #ifndef X_OLD_CLANG /** * Adapt a smart pointer to a typed memory block (unique_ptr or shared_ptr) @@ -429,19 +531,20 @@ namespace xt * * @param smart_ptr a smart pointer to a memory block of T[] * @param shape The desired shape + * @param l The desired memory layout * * @return xtensor_adaptor for memory */ - template - auto adapt_smart_ptr(P&& smart_ptr, const I(&shape)[N]) + template + auto adapt_smart_ptr(P&& smart_ptr, const I(&shape)[N], layout_type l = L) { using buffer_adaptor = xbuffer_adaptor>; std::array fshape = xtl::forward_sequence, decltype(shape)>(shape); - return xtensor_adaptor( - buffer_adaptor(smart_ptr.get(), compute_size(fshape), - std::forward

(smart_ptr)), - std::move(fshape) + return xtensor_adaptor( + buffer_adaptor(smart_ptr.get(), compute_size(fshape), std::forward

(smart_ptr)), + std::move(fshape), + l ); } @@ -487,19 +590,22 @@ namespace xt * @param data_ptr A pointer to a typed data block (e.g. double*) * @param shape The desired shape * @param smart_ptr A smart pointer to move or copy, in order to manage memory + * @param l The desired memory layout * * @return xtensor_adaptor on the memory */ - template - auto adapt_smart_ptr(P&& data_ptr, const I(&shape)[N], D&& smart_ptr) + template >)> + auto adapt_smart_ptr(P&& data_ptr, const I(&shape)[N], D&& smart_ptr, layout_type l = L) { using buffer_adaptor = xbuffer_adaptor>; std::array fshape = xtl::forward_sequence, decltype(shape)>(shape); - return xtensor_adaptor( + return xtensor_adaptor( buffer_adaptor(data_ptr, compute_size(fshape), std::forward(smart_ptr)), - std::move(fshape) + std::move(fshape), + l ); } #endif diff --git a/test/test_xadapt.cpp b/test/test_xadapt.cpp index ad9380766..9aa4b2dc8 100644 --- a/test/test_xadapt.cpp +++ b/test/test_xadapt.cpp @@ -430,6 +430,42 @@ namespace xt }; } + TEST(xarray_adaptor, smart_ptr) + { + auto data = std::vector{1,2,3,4,5,6,7,8}; + auto shared_buf = std::make_shared(data); + auto unique_buf = std::make_unique(data); + std::vector shape = {4, 2}; + + std::shared_ptr dptr(new double[8], std::default_delete()); + dptr.get()[2] = 2.1; + auto xdptr = adapt_smart_ptr(dptr, shape); + + if (XTENSOR_DEFAULT_LAYOUT == layout_type::row_major) + { + EXPECT_EQ(xdptr(1, 0), 2.1); + xdptr(3, 1) = 123.; + EXPECT_EQ(dptr.get()[7], 123.); + } + else + { + EXPECT_EQ(xdptr(2, 0), 2.1); + xdptr(3, 1) = 123.; + EXPECT_EQ(dptr.get()[7], 123.); + } + + EXPECT_EQ(shared_buf.use_count(), 1); + { + auto obj = adapt_smart_ptr(shared_buf.get()->buf.data(), shape, shared_buf); + EXPECT_EQ(shared_buf.use_count(), 2); + } + EXPECT_EQ(shared_buf.use_count(), 1); + + { + auto obj = adapt_smart_ptr(unique_buf.get()->buf.data(), shape, std::move(unique_buf)); + } + } + #ifndef X_OLD_CLANG TEST(xtensor_adaptor, smart_ptr) { From ae91091d0b005f3351a0fc46e3a95a336655ad50 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sat, 30 May 2020 13:31:17 -0700 Subject: [PATCH 033/606] Typo fix: "whis" -> "this" --- docs/source/container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/container.rst b/docs/source/container.rst index 4fcbacc34..3db830905 100644 --- a/docs/source/container.rst +++ b/docs/source/container.rst @@ -79,7 +79,7 @@ Let's use ``xtensor`` instead of ``xarray`` in the previous example: std::array shape = { 3, 2, 4 }; xt::xtensor a(shape); - // whis is equivalent to + // this is equivalent to // xt::xtensor a(shape); Or when using ``xtensor_fixed``: From 4d9581bbcff76ad63e47fe0766627386510f921e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sun, 31 May 2020 15:21:31 -0700 Subject: [PATCH 034/606] Typo fix: "tnesor" -> "tensor" and "Wether" -> "Whether". --- include/xtensor/xtensor_forward.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xtensor_forward.hpp b/include/xtensor/xtensor_forward.hpp index 4f3f1aab1..334cba871 100644 --- a/include/xtensor/xtensor_forward.hpp +++ b/include/xtensor/xtensor_forward.hpp @@ -175,7 +175,7 @@ namespace xt * @tparam T The value type of the elements. * @tparam FSH A xshape template shape. * @tparam L The layout_type of the tensor (default: XTENSOR_DEFAULT_LAYOUT). - * @tparam Sharable Wether the tnesor can be used in shared expression. + * @tparam Sharable Whether the tensor can be used in shared expression. */ template Date: Sun, 31 May 2020 20:09:21 -0700 Subject: [PATCH 035/606] Removing spurious "Python code" from a code block. --- docs/source/related.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/related.rst b/docs/source/related.rst index 9391519b1..2a526f9a5 100644 --- a/docs/source/related.rst +++ b/docs/source/related.rst @@ -63,8 +63,6 @@ Example 1: Use an algorithm of the C++ library on a numpy array in-place .. code:: - Python Code - import numpy as np import xtensor_python_test as xt From fc1f9022e90283fa0d17553c1fe73b46ac415ced Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 3 Jun 2020 06:49:09 +0200 Subject: [PATCH 036/606] Removed problematic function from doxygen parsing --- include/xtensor/xutils.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/xtensor/xutils.hpp b/include/xtensor/xutils.hpp index 72b2066f3..fda1b8395 100644 --- a/include/xtensor/xutils.hpp +++ b/include/xtensor/xutils.hpp @@ -201,6 +201,8 @@ namespace xt * accumulate implementation * *****************************/ + /// @cond DOXYGEN_INCLUDE_NOEXCEPT + namespace detail { template @@ -227,6 +229,8 @@ namespace xt return detail::accumulate_impl<0, F, R, T...>(std::forward(f), init, t); } + /// @endcond + /*************************** * argument implementation * ***************************/ From 1135ede7e465128be0534a961cb052ae8f361728 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 4 Jun 2020 14:36:10 +0200 Subject: [PATCH 037/606] Pin sphinx version on rtd --- docs/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/environment.yml b/docs/environment.yml index 590773d13..21a4fd065 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -5,3 +5,4 @@ channels: dependencies: - breathe + - sphinx=2.4.4 From eb690983a7364a9bb97caa8ebc5d40d720d2f0f0 Mon Sep 17 00:00:00 2001 From: Frederic Weidling Date: Thu, 4 Jun 2020 15:41:08 +0200 Subject: [PATCH 038/606] Fix xtensor::arange for unsigned step size --- include/xtensor/xbuilder.hpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/include/xtensor/xbuilder.hpp b/include/xtensor/xbuilder.hpp index 659171b54..20a11622e 100644 --- a/include/xtensor/xbuilder.hpp +++ b/include/xtensor/xbuilder.hpp @@ -297,6 +297,12 @@ namespace xt template using both_integer = xtl::conjunction, std::is_integral>; + template + using integer_with_signed_integer = xtl::conjunction, std::is_signed>; + + template + using integer_with_unsigned_integer = xtl::conjunction, std::is_unsigned>; + template >)> inline auto arange_impl(T start, T stop, S step = 1) noexcept { @@ -304,7 +310,7 @@ namespace xt return detail::make_xgenerator(detail::arange_generator(start, stop, step), {shape}); } - template )> + template )> inline auto arange_impl(T start, T stop, S step = 1) noexcept { bool empty_cond = (stop - start) / step <= 0; @@ -317,6 +323,18 @@ namespace xt return detail::make_xgenerator(detail::arange_generator(start, stop, step), {shape}); } + template )> + inline auto arange_impl(T start, T stop, S step = 1) noexcept + { + bool empty_cond = stop <= start; + std::size_t shape = 0; + if (!empty_cond) + { + shape = static_cast((stop - start + step - S(1)) / step); + } + return detail::make_xgenerator(detail::arange_generator(start, stop, step), { shape }); + } + template class fn_impl { From e35667ae010aa632eb68f8f60172f1ea4237e4bd Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 4 Jun 2020 09:11:12 -0700 Subject: [PATCH 039/606] Adding missing "a" to "in a shared expression". --- include/xtensor/xtensor_forward.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xtensor_forward.hpp b/include/xtensor/xtensor_forward.hpp index 334cba871..c2bfc452d 100644 --- a/include/xtensor/xtensor_forward.hpp +++ b/include/xtensor/xtensor_forward.hpp @@ -175,7 +175,7 @@ namespace xt * @tparam T The value type of the elements. * @tparam FSH A xshape template shape. * @tparam L The layout_type of the tensor (default: XTENSOR_DEFAULT_LAYOUT). - * @tparam Sharable Whether the tensor can be used in shared expression. + * @tparam Sharable Whether the tensor can be used in a shared expression. */ template Date: Fri, 5 Jun 2020 08:45:24 +0200 Subject: [PATCH 040/606] Improved var and stddev performance --- include/xtensor/xmath.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index 126817b31..88a39409c 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -2085,8 +2085,8 @@ namespace detail { XTL_REQUIRES(is_reducer_options, std::is_integral)> inline auto variance(E&& e, D const& ddof, EVS es = EVS()) { - decltype(auto) sc = detail::shared_forward(e); - return detail::mean_noaxis(square(sc - mean(sc, es)), ddof, es); + auto cached_mean = mean(e, es)(); + return detail::mean_noaxis(square(std::forward(e) - std::move(cached_mean)), ddof, es); } template Date: Fri, 5 Jun 2020 09:43:54 +0200 Subject: [PATCH 041/606] Documented fixed_shape weird bug on Windows --- docs/source/pitfall.rst | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/source/pitfall.rst b/docs/source/pitfall.rst index 32e2423a5..508c5f852 100644 --- a/docs/source/pitfall.rst +++ b/docs/source/pitfall.rst @@ -99,7 +99,7 @@ variance arguments ------------------ When ``variance`` is passed an expression and an integer parameter, this latter -is the axis along which the variance must be computed, but the degree of freedom: +is not the axis along which the variance must be computed, but the degree of freedom: .. code:: @@ -115,3 +115,20 @@ If you want to specify an axis, you need to pass an initializer list: std::cout << xt::variance(a, {1}) << std::endl; .. Outputs { 0.666667, 0.666667 } +fixed_shape on Windows +---------------------- + +Builder functions such as ``empty`` or ``ones`` accept an initializer list +as argument. If the elements of this list do not have the same type, a +curious compilation error may occur on Windows: + +.. code:: + + size_t N = 10ull; + xt::xarray ages = xt::empty({N, 4ul}); + + // error: cannot convert argument 1 from 'initializer list' + // to 'const xt::fixed_shape<> &' + +To avoid this compiler bug (for which we don't have a workaround), ensure +all the elements in the initializer list have the same type. From ca2323640d827cdefa143eb2dedb71897246f5f9 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Sun, 7 Jun 2020 01:00:01 +0200 Subject: [PATCH 042/606] Upgraded to xsimd 7.4.8 --- .azure-pipelines/azure-pipelines-win.yml | 2 +- README.md | 2 +- environment-dev.yml | 2 +- include/xtensor/xassign.hpp | 8 ++++---- test/test_xarray.cpp | 17 +++++++++++++++++ test/test_xtensor.cpp | 17 +++++++++++++++++ 6 files changed, 41 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 4598db9ad..de8af69b1 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -39,7 +39,7 @@ jobs: ninja ^ nlohmann_json ^ xtl==0.6.12 ^ - xsimd==7.4.6 ^ + xsimd==7.4.8 ^ python=3.6 conda list displayName: "Install conda packages" diff --git a/README.md b/README.md index 8fa618da7..dabfc3024 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| -| master | ^0.6.12 | ^7.4.6 | +| master | ^0.6.12 | ^7.4.8 | | 0.21.5 | ^0.6.12 | ^7.4.6 | | 0.21.4 | ^0.6.12 | ^7.4.6 | | 0.21.3 | ^0.6.9 | ^7.4.4 | diff --git a/environment-dev.yml b/environment-dev.yml index aa6a6b4fb..41cf4e747 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -4,5 +4,5 @@ channels: dependencies: - cmake - xtl=0.6.12 - - xsimd=7.4.6 + - xsimd=7.4.8 - nlohmann_json diff --git a/include/xtensor/xassign.hpp b/include/xtensor/xassign.hpp index 0b81d27af..b4f172b4d 100644 --- a/include/xtensor/xassign.hpp +++ b/include/xtensor/xassign.hpp @@ -295,16 +295,16 @@ namespace xt /** * Considering the assigment LHS = RHS, if the requested value type used for - * loading simd form RHS is not complex while LHS value_type is complex, + * loading simd from RHS is not complex while LHS value_type is complex, * the assignment fails. The reason is that SIMD batches of complex values cannot * be implicitly instanciated from batches of scalar values. * Making the constructor implicit does not fix the issue since in the end, * the assignment is done with vec.store(buffer) where vec is a batch of scalars * and buffer an array of complex. SIMD batches of scalars do not provide overloads - * of store that accept buffer of commplex values and that SHOULD NOT CHANGE. + * of store that accept buffer of complex values and that SHOULD NOT CHANGE. * Load and store overloads must accept SCALAR BUFFERS ONLY. * Therefore, the solution is to explicitly force the instantiation of complex - * batches in the assignment mechanism. A common situation tthat triggers this + * batches in the assignment mechanism. A common situation that triggers this * issue is: * xt::xarray rhs = { 1, 2, 3 }; * xt::xarray> lhs = rhs; @@ -953,7 +953,7 @@ namespace xt { for (std::size_t i = 0; i < simd_size; ++i) { - res_stepper.template store_simd(fct_stepper.template step_simd()); + res_stepper.store_simd(fct_stepper.template step_simd()); } for (std::size_t i = 0; i < simd_rest; ++i) { diff --git a/test/test_xarray.cpp b/test/test_xarray.cpp index 677387665..af525e1eb 100644 --- a/test/test_xarray.cpp +++ b/test/test_xarray.cpp @@ -361,4 +361,21 @@ namespace xt EXPECT_TRUE(std::is_destructible::value); EXPECT_TRUE(std::is_nothrow_destructible::value); } + + TEST(xarray, bool_container) + { + xt::xarray a{1, 0, 1, 0}, b{1, 1, 0, 0}; + + xt::xarray c = a & b; + EXPECT_TRUE(c(0)); + EXPECT_FALSE(c(1)); + EXPECT_FALSE(c(2)); + EXPECT_FALSE(c(3)); + + xt::xarray d = a | b; + EXPECT_TRUE(d(0)); + EXPECT_TRUE(d(1)); + EXPECT_TRUE(d(2)); + EXPECT_FALSE(d(3)); + } } diff --git a/test/test_xtensor.cpp b/test/test_xtensor.cpp index 2759bc041..f9c2fa9be 100644 --- a/test/test_xtensor.cpp +++ b/test/test_xtensor.cpp @@ -368,4 +368,21 @@ namespace xt EXPECT_TRUE(std::is_destructible::value); EXPECT_TRUE(std::is_nothrow_destructible::value); } + + TEST(xtensor, bool_container) + { + xt::xtensor a{1, 0, 1, 0}, b{1, 1, 0, 0}; + + xt::xtensor c = a & b; + EXPECT_TRUE(c(0)); + EXPECT_FALSE(c(1)); + EXPECT_FALSE(c(2)); + EXPECT_FALSE(c(3)); + + xt::xtensor d = a | b; + EXPECT_TRUE(d(0)); + EXPECT_TRUE(d(1)); + EXPECT_TRUE(d(2)); + EXPECT_FALSE(d(3)); + } } From 8f5ec05cd43de8dd90522beb5913a8ad1c41cfc9 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 16 Jun 2020 16:48:31 +0200 Subject: [PATCH 043/606] Add comments --- include/xtensor/xbuilder.hpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/include/xtensor/xbuilder.hpp b/include/xtensor/xbuilder.hpp index 20a11622e..56ece953f 100644 --- a/include/xtensor/xbuilder.hpp +++ b/include/xtensor/xbuilder.hpp @@ -674,11 +674,11 @@ namespace xt template using all_true = xtl::conjunction...>; - template + template struct concat_fixed_shape_impl; template - struct concat_fixed_shape_impl> + struct concat_fixed_shape_impl> { static_assert(X::size() == Y::size(), "Concatenation requires equisized shapes"); static_assert(axis < X::size(), "Concatenation requires a valid axis"); @@ -689,17 +689,17 @@ namespace xt : X::template get())...>; }; - template + template struct concat_fixed_shape; - template - struct concat_fixed_shape + template + struct concat_fixed_shape { using type = typename concat_fixed_shape_impl>::type; }; - template - struct concat_fixed_shape + template + struct concat_fixed_shape { using type = typename concat_fixed_shape::type>::type; }; @@ -775,7 +775,7 @@ namespace xt * xt::xarray a = {{1, 2, 3}}; * xt::xarray b = {{2, 3, 4}}; * xt::xarray c = xt::concatenate(xt::xtuple(a, b)); // => {{1, 2, 3}, - * {2, 3, 4}} + * // {2, 3, 4}} * xt::xarray d = xt::concatenate(xt::xtuple(a, b), 1); // => {{1, 2, 3, 2, 3, 4}} * \endcode */ @@ -787,7 +787,7 @@ namespace xt } template ::value>> - inline auto concatenate(std::tuple &&t) + inline auto concatenate(std::tuple &&t) { using shape_type = detail::concat_fixed_shape_t::shape_type...>; return detail::make_xgenerator(detail::concatenate_impl(std::move(t), axis), shape_type{}); @@ -826,10 +826,10 @@ namespace xt * xt::xarray a = {1, 2, 3}; * xt::xarray b = {5, 6, 7}; * xt::xarray s = xt::stack(xt::xtuple(a, b)); // => {{1, 2, 3}, - * {5, 6, 7}} + * // {5, 6, 7}} * xt::xarray t = xt::stack(xt::xtuple(a, b), 1); // => {{1, 5}, - * {2, 6}, - * {3, 7}} + * // {2, 6}, + * // {3, 7}} * \endcode */ template From 59a737cd1681e212a924f4660a4858e062c5d187 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 19 Jun 2020 11:15:54 +0200 Subject: [PATCH 044/606] How to print shape (#2071) [doc] How to print shape --- docs/source/getting_started.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 608fba68a..4f675a478 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -166,13 +166,19 @@ When compiled and run, this produces the following output: .. tip:: - To print the shape to the standard output you can use + To print the shape to the standard output you can use either: .. code-block:: cpp const auto& s = arr.shape(); std::copy(s.cbegin(), s.cend(), std::ostream_iterator(std::cout, " ")); + Or: + + .. code-block:: cpp + + std::cout << xt::adapt(arr.shape()); // with: #include "xtensor/xadapt.hpp" + Third example: index access --------------------------- From b3873da2ac4740568fa6517120f4c7acc8d17cb1 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 23 Jun 2020 22:58:47 +0200 Subject: [PATCH 045/606] Improved aliasing documentation --- docs/source/container.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/container.rst b/docs/source/container.rst index 3db830905..68640679b 100644 --- a/docs/source/container.rst +++ b/docs/source/container.rst @@ -140,8 +140,8 @@ Aliasing and temporaries ------------------------ In some cases, an expression should not be directly assigned to a container. Instead, it has to be assigned to a temporary variable before being copied -into the destination container. This occurs when the destination container is involved in the expression and has to be resized. This phenomenon is -known as *aliasing*. +into the destination container. A typical case where this happens is when the destination container is involved in the expression and has to be resized. +This phenomenon is known as *aliasing*. To prevent this, `xtensor` assigns the expression to a temporary variable before copying it. In the case of ``xarray``, this results in an extra dynamic memory allocation and copy. From a83a8f6be5b796faa2630ffa17416159fea822c4 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 24 Jun 2020 11:23:57 +0200 Subject: [PATCH 046/606] Added missing typedef in xscalar_stepper --- include/xtensor/xscalar.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/xtensor/xscalar.hpp b/include/xtensor/xscalar.hpp index 0c9e74cf8..83a1cb008 100644 --- a/include/xtensor/xscalar.hpp +++ b/include/xtensor/xscalar.hpp @@ -363,6 +363,7 @@ namespace xt typename storage_type::pointer>; using size_type = typename storage_type::size_type; using difference_type = typename storage_type::difference_type; + using shape_type = typename storage_type::shape_type; template using simd_return_type = xt_simd::simd_return_type; From 374145974db9df893427151c581086a8be1fb7e5 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Wed, 1 Jul 2020 15:27:07 +0200 Subject: [PATCH 047/606] Fix doc --- include/xtensor/xaccessible.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/xtensor/xaccessible.hpp b/include/xtensor/xaccessible.hpp index 8d8c6b3b2..600ddd0b2 100644 --- a/include/xtensor/xaccessible.hpp +++ b/include/xtensor/xaccessible.hpp @@ -23,8 +23,7 @@ namespace xt * The xaccessible class implements constant access methods common to all expressions. * * @tparam D The derived type, i.e. the inheriting class for which xconst_accessible - * - * + * provides the interface. */ template class xconst_accessible From e04b802353a57ce2bc6ca7173b406674ea799ee1 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2020 15:23:37 +0200 Subject: [PATCH 048/606] Add xchunked_array (#2076) Add xchunked_array --- include/xtensor/xchunked_array.hpp | 87 ++++++++++++++++++++++++++++++ test/CMakeLists.txt | 1 + test/test_xchunked_array.cpp | 25 +++++++++ 3 files changed, 113 insertions(+) create mode 100644 include/xtensor/xchunked_array.hpp create mode 100644 test/test_xchunked_array.cpp diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp new file mode 100644 index 000000000..896f80bb9 --- /dev/null +++ b/include/xtensor/xchunked_array.hpp @@ -0,0 +1,87 @@ +#include +#include +#include "xarray.hpp" + +namespace xt +{ + template + class xchunked_array + { + public: + + using const_reference = typename chunk_type::const_reference; + + template + inline const_reference operator()(Idxs... idxs) const + { + auto chunk_indexes_packed = get_chunk_indexes(std::make_index_sequence(), idxs...); + auto chunk_indexes = unpack(chunk_indexes_packed); + auto indexes_of_chunk(std::get<0>(chunk_indexes)); + auto indexes_in_chunk(std::get<1>(chunk_indexes)); + chunk_type chunk = m_chunks.element(indexes_of_chunk.cbegin(), indexes_of_chunk.cend()); + const_reference val = chunk.element(indexes_in_chunk.cbegin(), indexes_in_chunk.cend()); + int di = 0; + for (auto index_of_chunk: indexes_of_chunk) + { + auto index_in_chunk = indexes_in_chunk[di]; + di++; + } + return val; + } + + + xchunked_array(std::vector shape, std::vector chunks): + m_shape(shape), + m_chunk_shape(chunks) + { + std::vector shape_chunk(shape.size()); + size_t di = 0; + for (auto s: shape) + { + size_t chunk_nb = s / chunks[di]; + if (s % chunks[di] > 0) + chunk_nb += 1; // edge chunk + shape_chunk[di] = chunk_nb; + di++; + } + for (auto s: chunks) + m_chunks.resize(shape_chunk); + } + + private: + + template + std::tuple get_chunk_indexes_in_dimension(Dim dim, Idx idx) const + { + size_t index_of_chunk = idx / m_chunk_shape[dim]; + size_t index_in_chunk = idx - index_of_chunk * m_chunk_shape[dim]; + return std::make_tuple(index_of_chunk, index_in_chunk); + } + + template + std::array, sizeof...(Idxs)> + get_chunk_indexes(std::index_sequence, Idxs... idxs) const + { + std::array, sizeof...(Idxs)> chunk_indexes = {{get_chunk_indexes_in_dimension(dims, idxs)...}}; + return chunk_indexes; + } + + template + std::tuple, std::array> unpack(std::array &arr) const + { + std::array arr0; + std::array arr1; + for (size_t i = 0; i < N; ++i) + { + arr0[i] = std::get<0>(arr[i]); + arr1[i] = std::get<1>(arr[i]); + } + return std::make_tuple(arr0, arr1); + } + + xt::xarray m_chunks; + std::vector m_shape; + std::vector m_chunk_shape; + }; + +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2184a8a78..7ed9a876a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -220,6 +220,7 @@ set(XTENSOR_TESTS test_extended_xmath_reducers.cpp test_extended_xhistogram.cpp test_extended_xsort.cpp + test_xchunked_array.cpp ) if(nlohmann_json_FOUND) diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp new file mode 100644 index 000000000..1210eeabc --- /dev/null +++ b/test/test_xchunked_array.cpp @@ -0,0 +1,25 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include "gtest/gtest.h" +#include "xtensor/xchunked_array.hpp" + +namespace xt +{ + using chunked_array = xt::xchunked_array>; + + TEST(xchunked_array, indexed_access) + { + chunked_array a( + {10, 10, 10}, + {2, 3, 4} + ); + a(3, 9, 8); + } +} From 0aea191c7bd4033b99de8b78e4418de8a1ccd241 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 3 Jul 2020 18:09:54 +0200 Subject: [PATCH 049/606] Fix doc --- docs/source/external-structures.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/external-structures.rst b/docs/source/external-structures.rst index b7e7d73e6..4df8f34bd 100644 --- a/docs/source/external-structures.rst +++ b/docs/source/external-structures.rst @@ -271,7 +271,7 @@ The following definitions are required: .. code:: template - struct xcontainer_inner_type> + struct xcontainer_inner_types> { using temporary_type = xarray; }; @@ -299,7 +299,8 @@ and to define a bunch of typedefs. public: - using self_type = table; + using self_type = table_adaptor; + using semantic_base = xcontainer_semantic; using value_type = T; using reference = T&; @@ -314,7 +315,7 @@ and to define a bunch of typedefs. using shape_type = inner_shape_type; using strides_type = inner_strides_type; - using iterable_base = xexpression_iterable; + using iterable_base = xiterable; using stepper = typename iterable_base::stepper; using const_stepper = typename iterable_base::const_stepper; }; @@ -353,7 +354,6 @@ constructor and assign operator. template table_adaptor(const xexpression& e) - : base_type() { semantic_base::assign(e); } From 83ae7fce2cc183458c98cbe00d1637ad1d3eabc5 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2020 15:55:54 +0200 Subject: [PATCH 050/606] Inherit xconst_accessible --- include/xtensor/xchunked_array.hpp | 121 ++++++++++++++++++++++------- test/test_xchunked_array.cpp | 18 +++-- 2 files changed, 105 insertions(+), 34 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 896f80bb9..4a09f847d 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -5,69 +5,95 @@ namespace xt { template - class xchunked_array + class xchunked_array: public xt::xaccessible> { public: using const_reference = typename chunk_type::const_reference; + using reference = typename chunk_type::reference; template inline const_reference operator()(Idxs... idxs) const { - auto chunk_indexes_packed = get_chunk_indexes(std::make_index_sequence(), idxs...); - auto chunk_indexes = unpack(chunk_indexes_packed); - auto indexes_of_chunk(std::get<0>(chunk_indexes)); - auto indexes_in_chunk(std::get<1>(chunk_indexes)); - chunk_type chunk = m_chunks.element(indexes_of_chunk.cbegin(), indexes_of_chunk.cend()); - const_reference val = chunk.element(indexes_in_chunk.cbegin(), indexes_in_chunk.cend()); - int di = 0; - for (auto index_of_chunk: indexes_of_chunk) - { - auto index_in_chunk = indexes_in_chunk[di]; - di++; - } - return val; + auto ii = get_indexes(idxs...); + auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); + return chunk.element(ii.second.cbegin(), ii.second.cend()); } + template + inline reference operator()(Idxs... idxs) + { + auto ii = get_indexes(idxs...); + auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); + return chunk.element(ii.second.cbegin(), ii.second.cend()); + } - xchunked_array(std::vector shape, std::vector chunks): + template + xchunked_array(S shape, S chunk_shape): m_shape(shape), - m_chunk_shape(chunks) + m_chunk_shape(chunk_shape) { std::vector shape_chunk(shape.size()); size_t di = 0; for (auto s: shape) { - size_t chunk_nb = s / chunks[di]; - if (s % chunks[di] > 0) + size_t chunk_nb = s / chunk_shape[di]; + if (s % chunk_shape[di] > 0) chunk_nb += 1; // edge chunk shape_chunk[di] = chunk_nb; di++; } - for (auto s: chunks) m_chunks.resize(shape_chunk); + for (auto& c: m_chunks) + c.resize(chunk_shape); + } + + reference operator[](const xindex& index) + { + reference el = element(index.cbegin(), index.cend()); + return el; + } + + const_reference operator[](const xindex& index) const + { + const_reference const_el = element(index.cbegin(), index.cend()); + return const_el; } private: - template - std::tuple get_chunk_indexes_in_dimension(Dim dim, Idx idx) const + xt::xarray m_chunks; + typename chunk_type::shape_type m_shape; + typename chunk_type::shape_type m_chunk_shape; + + template + inline std::pair, std::array> get_indexes(Idxs... idxs) const + { + auto chunk_indexes_packed = get_chunk_indexes(std::make_index_sequence(), idxs...); + auto chunk_indexes = unpack(chunk_indexes_packed); + auto indexes_of_chunk = chunk_indexes.first; + auto indexes_in_chunk = chunk_indexes.second; + return std::make_pair(indexes_of_chunk, indexes_in_chunk); + } + + template + std::pair get_chunk_indexes_in_dimension(size_t dim, Idx idx) const { size_t index_of_chunk = idx / m_chunk_shape[dim]; size_t index_in_chunk = idx - index_of_chunk * m_chunk_shape[dim]; - return std::make_tuple(index_of_chunk, index_in_chunk); + return std::make_pair(index_of_chunk, index_in_chunk); } template - std::array, sizeof...(Idxs)> + std::array, sizeof...(Idxs)> get_chunk_indexes(std::index_sequence, Idxs... idxs) const { - std::array, sizeof...(Idxs)> chunk_indexes = {{get_chunk_indexes_in_dimension(dims, idxs)...}}; + std::array, sizeof...(Idxs)> chunk_indexes = {{get_chunk_indexes_in_dimension(dims, idxs)...}}; return chunk_indexes; } template - std::tuple, std::array> unpack(std::array &arr) const + std::pair, std::array> unpack(std::array &arr) const { std::array arr0; std::array arr1; @@ -76,12 +102,49 @@ namespace xt arr0[i] = std::get<0>(arr[i]); arr1[i] = std::get<1>(arr[i]); } - return std::make_tuple(arr0, arr1); + return std::make_pair(arr0, arr1); } - xt::xarray m_chunks; - std::vector m_shape; - std::vector m_chunk_shape; + template + inline std::pair, std::vector> get_indexes_dynamic(It first, It last) const + { + std::vector indexes_of_chunk; + std::vector indexes_in_chunk; + std::pair chunk_index; + size_t dim = 0; + for (auto it = first; it != last; ++it) + { + chunk_index = get_chunk_indexes_in_dimension(dim, *it); + indexes_of_chunk.push_back(chunk_index.first); + indexes_in_chunk.push_back(chunk_index.second); + dim++; + } + return std::make_pair(indexes_of_chunk, indexes_in_chunk); + } + + template + inline reference element(It first, It last) + { + auto ii = get_indexes_dynamic(first, last); + auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); + return chunk.element(ii.second.begin(), ii.second.end()); + } + + template + inline const_reference element(It first, It last) const + { + auto ii = get_indexes_dynamic(first, last); + auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); + return chunk.element(ii.second.begin(), ii.second.end()); + } }; + template + struct xcontainer_inner_types> + { + using temporary_type = xarray; + using const_reference = typename chunk_type::const_reference; + using reference = typename chunk_type::reference; + using size_type = std::size_t; + }; } diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 1210eeabc..ef4589b33 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -16,10 +16,18 @@ namespace xt TEST(xchunked_array, indexed_access) { - chunked_array a( - {10, 10, 10}, - {2, 3, 4} - ); - a(3, 9, 8); + std::vector shape = {10, 10, 10}; + std::vector chunk_shape = {2, 3, 4}; + chunked_array a(shape, chunk_shape); + + std::vector idx = {3, 9, 8}; + + a[idx] = 4.; + ASSERT_EQ(a[idx], 4.); + ASSERT_EQ(a(3, 9, 8), 4.); + + a(3, 9, 8) = 5.; + ASSERT_EQ(a(3, 9, 8), 5.); + ASSERT_EQ(a[idx], 5.); } } From 5f963a0f243511a827db5a19816e5e2e99ef0c58 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 6 Jul 2020 13:42:43 +0200 Subject: [PATCH 051/606] Inherit xiterable --- include/xtensor/xchunked_array.hpp | 107 ++++++++++++++++++++++++----- test/test_xchunked_array.cpp | 21 ++++-- 2 files changed, 103 insertions(+), 25 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 4a09f847d..019533cc5 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -5,12 +5,40 @@ namespace xt { template - class xchunked_array: public xt::xaccessible> + class xchunked_array: public xt::xaccessible>, + public xt::xiterable> { public: using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; + using self_type = xchunked_array; + using iterable_base = xconst_iterable; + using const_stepper = typename iterable_base::const_stepper; + using stepper = typename iterable_base::stepper; + using inner_types = xcontainer_inner_types; + using size_type = typename inner_types::size_type; + using storage_type = typename inner_types::storage_type; + using value_type = typename storage_type::value_type; + using pointer = value_type*; + using const_pointer = const value_type*; + using difference_type = std::ptrdiff_t; + using shape_type = typename chunk_type::shape_type; + + template + const_stepper stepper_begin(const O& shape) const noexcept; + template + const_stepper stepper_end(const O& shape, layout_type) const noexcept; + + template + stepper stepper_begin(const O& shape) noexcept; + template + stepper stepper_end(const O& shape, layout_type) noexcept; + + const shape_type& shape() const + { + return m_shape; + } template inline const_reference operator()(Idxs... idxs) const @@ -60,11 +88,27 @@ namespace xt return const_el; } + template + inline reference element(It first, It last) + { + auto ii = get_indexes_dynamic(first, last); + auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); + return chunk.element(ii.second.begin(), ii.second.end()); + } + + template + inline const_reference element(It first, It last) const + { + auto ii = get_indexes_dynamic(first, last); + auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); + return chunk.element(ii.second.begin(), ii.second.end()); + } + private: xt::xarray m_chunks; - typename chunk_type::shape_type m_shape; - typename chunk_type::shape_type m_chunk_shape; + shape_type m_shape; + shape_type m_chunk_shape; template inline std::pair, std::array> get_indexes(Idxs... idxs) const @@ -121,22 +165,6 @@ namespace xt } return std::make_pair(indexes_of_chunk, indexes_in_chunk); } - - template - inline reference element(It first, It last) - { - auto ii = get_indexes_dynamic(first, last); - auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); - return chunk.element(ii.second.begin(), ii.second.end()); - } - - template - inline const_reference element(It first, It last) const - { - auto ii = get_indexes_dynamic(first, last); - auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); - return chunk.element(ii.second.begin(), ii.second.end()); - } }; template @@ -146,5 +174,46 @@ namespace xt using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; using size_type = std::size_t; + using storage_type = chunk_type; + }; + + template + struct xiterable_inner_types> + { + using inner_shape_type = typename chunk_type::shape_type; + using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; }; + + template + template + inline auto xchunked_array::stepper_begin(const O& shape) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset); + } + + template + template + inline auto xchunked_array::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset, true); + } + + template + template + inline auto xchunked_array::stepper_begin(const O& shape) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset); + } + + template + template + inline auto xchunked_array::stepper_end(const O& shape, layout_type) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset, true); + } } diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index ef4589b33..936091d74 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -21,13 +21,22 @@ namespace xt chunked_array a(shape, chunk_shape); std::vector idx = {3, 9, 8}; + double v; - a[idx] = 4.; - ASSERT_EQ(a[idx], 4.); - ASSERT_EQ(a(3, 9, 8), 4.); + v = 1.; + a[idx] = v; + ASSERT_EQ(a[idx], v); + ASSERT_EQ(a(3, 9, 8), v); - a(3, 9, 8) = 5.; - ASSERT_EQ(a(3, 9, 8), 5.); - ASSERT_EQ(a[idx], 5.); + v = 2.; + a(3, 9, 8) = v; + ASSERT_EQ(a(3, 9, 8), v); + ASSERT_EQ(a[idx], v); + + v = 3.; + for (auto& it: a) + it = v; + for (auto it: a) + ASSERT_EQ(it, v); } } From 20d2642126ae2b5d81cfce0b179379f418218764 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 7 Jul 2020 22:03:53 +0200 Subject: [PATCH 052/606] Inherit from xcontainer_semantic (#2083) Inherit from xcontainer_semantic --- include/xtensor/xchunked_array.hpp | 116 ++++++++++++++++++++++++----- test/test_xchunked_array.cpp | 25 +++++++ 2 files changed, 122 insertions(+), 19 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 019533cc5..42f90d38f 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -1,18 +1,47 @@ +#ifndef XTENSOR_CHUNKED_ARRAY_HPP +#define XTENSOR_CHUNKED_ARRAY_HPP + #include #include + #include "xarray.hpp" +#include "xnoalias.hpp" +#include "xstrided_view.hpp" namespace xt { template - class xchunked_array: public xt::xaccessible>, - public xt::xiterable> + class xchunked_array; + + template + struct xcontainer_inner_types> + { + using const_reference = typename chunk_type::const_reference; + using reference = typename chunk_type::reference; + using size_type = std::size_t; + using storage_type = chunk_type; + using temporary_type = xchunked_array; + }; + + template + struct xiterable_inner_types> + { + using inner_shape_type = typename chunk_type::shape_type; + using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; + }; + + template + class xchunked_array: public xaccessible>, + public xiterable>, + public xcontainer_semantic> { public: using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; using self_type = xchunked_array; + using semantic_base = xcontainer_semantic; using iterable_base = xconst_iterable; using const_stepper = typename iterable_base::const_stepper; using stepper = typename iterable_base::stepper; @@ -24,6 +53,7 @@ namespace xt using const_pointer = const value_type*; using difference_type = std::ptrdiff_t; using shape_type = typename chunk_type::shape_type; + using temporary_type = typename inner_types::temporary_type; template const_stepper stepper_begin(const O& shape) const noexcept; @@ -76,6 +106,49 @@ namespace xt c.resize(chunk_shape); } + xchunked_array(const xchunked_array&) = default; + xchunked_array& operator=(const xchunked_array&) = default; + + xchunked_array(xchunked_array&&) = default; + xchunked_array& operator=(xchunked_array&&) = default; + + template + xchunked_array(const xexpression& e) + { + const auto& sh = e.derived_cast().shape(); + resize_container(m_shape, sh.size()); + std::copy(sh.begin(), sh.end(), m_shape.begin()); + m_chunk_shape = m_shape; + // Naive implementation to refine later + m_chunk_shape[0] = std::min(size_type(10), m_shape[0]); + size_type nb_chunks = m_shape[0] / m_chunk_shape[0]; + bool additional_chunk = m_shape[0] % m_chunk_shape[0] > 0; + if (additional_chunk) + { + m_chunks.resize({nb_chunks + 1u}); + } + else + { + m_chunks.resize({nb_chunks}); + } + for (size_type i = 0; i < nb_chunks; ++i) + { + noalias(m_chunks(i)) = strided_view(e.derived_cast(), + {range(i * m_chunk_shape[0], (i + 1u) * m_chunk_shape[0]), ellipsis()}); + } + if (additional_chunk) + { + noalias(m_chunks(nb_chunks)) = strided_view(e.derived_cast(), + {range(nb_chunks * m_chunk_shape[0], m_shape[0]), ellipsis()}); + } + } + + template + self_type& operator=(const xexpression& e) + { + return semantic_base::operator=(e); + } + reference operator[](const xindex& index) { reference el = element(index.cbegin(), index.cend()); @@ -106,7 +179,7 @@ namespace xt private: - xt::xarray m_chunks; + xarray m_chunks; shape_type m_shape; shape_type m_chunk_shape; @@ -165,26 +238,28 @@ namespace xt } return std::make_pair(indexes_of_chunk, indexes_in_chunk); } - }; - template - struct xcontainer_inner_types> - { - using temporary_type = xarray; - using const_reference = typename chunk_type::const_reference; - using reference = typename chunk_type::reference; - using size_type = std::size_t; - using storage_type = chunk_type; - }; + size_type dimension() const + { + return shape().size(); + } - template - struct xiterable_inner_types> - { - using inner_shape_type = typename chunk_type::shape_type; - using const_stepper = xindexed_stepper, true>; - using stepper = xindexed_stepper, false>; + template + bool broadcast_shape(const S& s) const + { + // Available in "xtensor/xtrides.hpp" + return broadcast_shape(shape(), s); + } + + template + bool is_trivial_broadcast(const S& str) const noexcept + { + return false; + } }; + + template template inline auto xchunked_array::stepper_begin(const O& shape) const noexcept -> const_stepper @@ -217,3 +292,6 @@ namespace xt return stepper(this, offset, true); } } + +#endif + diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 936091d74..449fb226c 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -8,6 +8,8 @@ ****************************************************************************/ #include "gtest/gtest.h" + +#include "xtensor/xbroadcast.hpp" #include "xtensor/xchunked_array.hpp" namespace xt @@ -39,4 +41,27 @@ namespace xt for (auto it: a) ASSERT_EQ(it, v); } + + TEST(xchunked_array, assign_expression) + { + std::vector shape = {2, 2, 2}; + std::vector chunk_shape = {2, 3, 4}; + chunked_array a(shape, chunk_shape); + + a = xt::broadcast(3., a.shape()); + for (const auto& v: a) + { + EXPECT_EQ(v, 3.); + } + + std::vector shape2 = {32, 10, 10}; + chunked_array a2(shape2, chunk_shape); + + a2 = xt::broadcast(3., a2.shape()); + for (const auto& v: a2) + { + EXPECT_EQ(v, 3.); + } + + } } From e7fdcf8b2aea9860424f1268f6ab16dd466563e2 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Wed, 8 Jul 2020 10:30:22 +0200 Subject: [PATCH 053/606] Fix assignment operator --- include/xtensor/xchunked_array.hpp | 35 +++++++++++++-------- test/test_xchunked_array.cpp | 49 +++++++++++++++++------------- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 42f90d38f..6b063b263 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -54,6 +54,8 @@ namespace xt using difference_type = std::ptrdiff_t; using shape_type = typename chunk_type::shape_type; using temporary_type = typename inner_types::temporary_type; + using bool_load_type = xt::bool_load_type; + static constexpr layout_type static_layout = layout_type::dynamic; template const_stepper stepper_begin(const O& shape) const noexcept; @@ -70,6 +72,16 @@ namespace xt return m_shape; } + inline layout_type layout() const noexcept + { + return static_layout; + } + + inline bool is_contiguous() const noexcept + { + return false; + } + template inline const_reference operator()(Idxs... idxs) const { @@ -177,6 +189,17 @@ namespace xt return chunk.element(ii.second.begin(), ii.second.end()); } + size_type dimension() const + { + return shape().size(); + } + + template + bool broadcast_shape(S& s, bool reuse_cache = false) const + { + return xt::broadcast_shape(shape(), s); + } + private: xarray m_chunks; @@ -239,18 +262,6 @@ namespace xt return std::make_pair(indexes_of_chunk, indexes_in_chunk); } - size_type dimension() const - { - return shape().size(); - } - - template - bool broadcast_shape(const S& s) const - { - // Available in "xtensor/xtrides.hpp" - return broadcast_shape(shape(), s); - } - template bool is_trivial_broadcast(const S& str) const noexcept { diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 449fb226c..0dd39d2c4 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -23,45 +23,52 @@ namespace xt chunked_array a(shape, chunk_shape); std::vector idx = {3, 9, 8}; - double v; + double val; - v = 1.; - a[idx] = v; - ASSERT_EQ(a[idx], v); - ASSERT_EQ(a(3, 9, 8), v); + val = 1.; + a[idx] = val; + ASSERT_EQ(a[idx], val); + ASSERT_EQ(a(3, 9, 8), val); - v = 2.; - a(3, 9, 8) = v; - ASSERT_EQ(a(3, 9, 8), v); - ASSERT_EQ(a[idx], v); + val = 2.; + a(3, 9, 8) = val; + ASSERT_EQ(a(3, 9, 8), val); + ASSERT_EQ(a[idx], val); - v = 3.; + val = 3.; for (auto& it: a) - it = v; + it = val; for (auto it: a) - ASSERT_EQ(it, v); + ASSERT_EQ(it, val); } TEST(xchunked_array, assign_expression) { - std::vector shape = {2, 2, 2}; - std::vector chunk_shape = {2, 3, 4}; - chunked_array a(shape, chunk_shape); + std::vector shape1 = {2, 2, 2}; + std::vector chunk_shape1 = {2, 3, 4}; + chunked_array a1(shape1, chunk_shape1); + double val; - a = xt::broadcast(3., a.shape()); - for (const auto& v: a) + val = 3.; + a1 = xt::broadcast(val, a1.shape()); + for (const auto& v: a1) { - EXPECT_EQ(v, 3.); + EXPECT_EQ(v, val); } std::vector shape2 = {32, 10, 10}; - chunked_array a2(shape2, chunk_shape); + chunked_array a2(shape2, chunk_shape1); - a2 = xt::broadcast(3., a2.shape()); + a2 = xt::broadcast(val, a2.shape()); for (const auto& v: a2) { - EXPECT_EQ(v, 3.); + EXPECT_EQ(v, val); } + a2 += a2; + for (const auto& v: a2) + { + EXPECT_EQ(v, 2. * val); + } } } From 88aa388ff8381d9426bc0bc4ccf345866c090500 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 10 Jul 2020 12:16:15 +0200 Subject: [PATCH 054/606] Add constructor from xexpression and chunk_shape --- include/xtensor/xchunked_array.hpp | 86 +++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 6b063b263..b5a9951c9 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -99,23 +99,9 @@ namespace xt } template - xchunked_array(S shape, S chunk_shape): - m_shape(shape), - m_chunk_shape(chunk_shape) + xchunked_array(S shape, S chunk_shape) { - std::vector shape_chunk(shape.size()); - size_t di = 0; - for (auto s: shape) - { - size_t chunk_nb = s / chunk_shape[di]; - if (s % chunk_shape[di] > 0) - chunk_nb += 1; // edge chunk - shape_chunk[di] = chunk_nb; - di++; - } - m_chunks.resize(shape_chunk); - for (auto& c: m_chunks) - c.resize(chunk_shape); + resize(shape, chunk_shape); } xchunked_array(const xchunked_array&) = default; @@ -155,6 +141,49 @@ namespace xt } } + template + xchunked_array(const xexpression& e, S chunk_shape) + { + const auto& shape = e.derived_cast().shape(); + resize_container(m_shape, shape.size()); + std::copy(shape.begin(), shape.end(), m_shape.begin()); + m_chunk_shape = chunk_shape; + resize(m_shape, m_chunk_shape); + shape_type ic(dimension()); // index of chunk, initialized to 0... + xstrided_slice_vector sv; // element slice corresponding to chunk + size_t di = 0; // dimension index + // initialize slice + for (auto i: ic) + { + sv.push_back(range(0, m_chunk_shape[di])); + di++; + } + for (auto& chunk: m_chunks) + { + noalias(chunk) = strided_view(e.derived_cast(), sv); + di = 0; + while (true) + { + if (ic[di] + 1 == m_chunks.shape()[di]) + { + ic[di] = 0; + sv[di] = range(0, m_chunk_shape[di]); + if (di + 1 == dimension()) + break; + else + di++; + + } + else + { + ic[di] += 1; + sv[di] = range(ic[di] * m_chunk_shape[di], (ic[di] + 1) * m_chunk_shape[di]); + break; + } + } + } + } + template self_type& operator=(const xexpression& e) { @@ -206,6 +235,29 @@ namespace xt shape_type m_shape; shape_type m_chunk_shape; + template + void resize(S& shape, S& chunk_shape) + { + // compute chunk number in each dimension (shape_of_chunks) + std::vector shape_of_chunks(shape.size()); + size_t di = 0; + for (auto s: shape) + { + size_t cn = s / chunk_shape[di]; + if (s % chunk_shape[di] > 0) + cn += 1; // edge chunk + shape_of_chunks[di] = cn; + di++; + } + // resize the xarray of chunks + m_chunks.resize(shape_of_chunks); + // resize each chunk + for (auto& c: m_chunks) + c.resize(chunk_shape); + m_shape = shape; + m_chunk_shape = chunk_shape; + } + template inline std::pair, std::array> get_indexes(Idxs... idxs) const { @@ -269,8 +321,6 @@ namespace xt } }; - - template template inline auto xchunked_array::stepper_begin(const O& shape) const noexcept -> const_stepper From 671638b3c96c6863a569424f91f72dc69efdbd5b Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 15 Jul 2020 12:45:56 +0200 Subject: [PATCH 055/606] Upgraded to xtl 0.6.15 --- README.md | 2 +- environment-dev.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dabfc3024..bfb1889be 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| -| master | ^0.6.12 | ^7.4.8 | +| master | ^0.6.15 | ^7.4.8 | | 0.21.5 | ^0.6.12 | ^7.4.6 | | 0.21.4 | ^0.6.12 | ^7.4.6 | | 0.21.3 | ^0.6.9 | ^7.4.4 | diff --git a/environment-dev.yml b/environment-dev.yml index 41cf4e747..1393e6ba5 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -3,6 +3,6 @@ channels: - conda-forge dependencies: - cmake - - xtl=0.6.12 + - xtl=0.6.15 - xsimd=7.4.8 - nlohmann_json From 7c796c88259e84596af1f06485191a434d516c90 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Wed, 15 Jul 2020 13:22:36 +0200 Subject: [PATCH 056/606] Add test --- include/xtensor/xchunked_array.hpp | 36 +++++++++++++++++------------- test/test_xchunked_array.cpp | 13 +++++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index b5a9951c9..a9c3eaa45 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -158,29 +158,35 @@ namespace xt sv.push_back(range(0, m_chunk_shape[di])); di++; } + size_t ci = 0; for (auto& chunk: m_chunks) { noalias(chunk) = strided_view(e.derived_cast(), sv); - di = 0; - while (true) + bool last_chunk = ci == m_chunks.size() - 1; + if (!last_chunk) { - if (ic[di] + 1 == m_chunks.shape()[di]) + di = 0; + while (true) { - ic[di] = 0; - sv[di] = range(0, m_chunk_shape[di]); - if (di + 1 == dimension()) - break; + if (ic[di] + 1 == m_chunks.shape()[di]) + { + ic[di] = 0; + sv[di] = range(0, m_chunk_shape[di]); + if (di + 1 == dimension()) + break; + else + di++; + + } else - di++; - - } - else - { - ic[di] += 1; - sv[di] = range(ic[di] * m_chunk_shape[di], (ic[di] + 1) * m_chunk_shape[di]); - break; + { + ic[di] += 1; + sv[di] = range(ic[di] * m_chunk_shape[di], (ic[di] + 1) * m_chunk_shape[di]); + break; + } } } + ci += 1; } } diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 0dd39d2c4..507b26aed 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -70,5 +70,18 @@ namespace xt { EXPECT_EQ(v, 2. * val); } + + xt::xarray a3 + {{1., 2., 3.}, + {4., 5., 6.}, + {7., 8., 9.}}; + std::vector chunk_shape4 = {2, 2}; + auto a4 = chunked_array(a3, chunk_shape4); + double i = 1.; + for (const auto& v: a4) + { + EXPECT_EQ(v, i); + i += 1.; + } } } From 162548dc2c260bfad9c028eec3de8d092ca549ef Mon Sep 17 00:00:00 2001 From: David Brochart Date: Wed, 15 Jul 2020 14:20:09 +0200 Subject: [PATCH 057/606] Fix chunk layout --- include/xtensor/xchunked_array.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index a9c3eaa45..2e94bb64b 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -165,17 +165,17 @@ namespace xt bool last_chunk = ci == m_chunks.size() - 1; if (!last_chunk) { - di = 0; + di = dimension() - 1; while (true) { if (ic[di] + 1 == m_chunks.shape()[di]) { ic[di] = 0; sv[di] = range(0, m_chunk_shape[di]); - if (di + 1 == dimension()) + if (di == 0) break; else - di++; + di--; } else From ef8ff77942122a4f57dbc91b7a5c56efac9b5ff0 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 16 Jul 2020 16:34:20 +0200 Subject: [PATCH 058/606] Copy constructor gets expression's chunk_shape if it is chunked (#2092) Copy constructor gets expression's chunk_shape if it is chunked --- include/xtensor/xchunked_array.hpp | 81 ++++++++++++++++++------------ test/test_xchunked_array.cpp | 23 +++++++++ 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 2e94bb64b..ba8ada3d5 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -10,6 +10,43 @@ namespace xt { + namespace detail + { + // Workaround for VS2015 + template + using try_chunk_shape = decltype(std::declval().chunk_shape()); + + template class OP, class = void> + struct chunk_helper_impl + { + static const auto& chunk_shape(const xexpression& e) + { + return e.derived_cast().shape(); + } + using is_chunked = std::false_type; + }; + + template class OP> + struct chunk_helper_impl>> + { + static const auto& chunk_shape(const xexpression& e) + { + return e.derived_cast().chunk_shape(); + } + using is_chunked = std::true_type; + }; + + template + using chunk_helper = chunk_helper_impl; + } + + template + constexpr bool is_chunked(const xexpression& e) + { + using return_type = typename detail::chunk_helper::is_chunked; + return return_type::value; + } + template class xchunked_array; @@ -110,37 +147,6 @@ namespace xt xchunked_array(xchunked_array&&) = default; xchunked_array& operator=(xchunked_array&&) = default; - template - xchunked_array(const xexpression& e) - { - const auto& sh = e.derived_cast().shape(); - resize_container(m_shape, sh.size()); - std::copy(sh.begin(), sh.end(), m_shape.begin()); - m_chunk_shape = m_shape; - // Naive implementation to refine later - m_chunk_shape[0] = std::min(size_type(10), m_shape[0]); - size_type nb_chunks = m_shape[0] / m_chunk_shape[0]; - bool additional_chunk = m_shape[0] % m_chunk_shape[0] > 0; - if (additional_chunk) - { - m_chunks.resize({nb_chunks + 1u}); - } - else - { - m_chunks.resize({nb_chunks}); - } - for (size_type i = 0; i < nb_chunks; ++i) - { - noalias(m_chunks(i)) = strided_view(e.derived_cast(), - {range(i * m_chunk_shape[0], (i + 1u) * m_chunk_shape[0]), ellipsis()}); - } - if (additional_chunk) - { - noalias(m_chunks(nb_chunks)) = strided_view(e.derived_cast(), - {range(nb_chunks * m_chunk_shape[0], m_shape[0]), ellipsis()}); - } - } - template xchunked_array(const xexpression& e, S chunk_shape) { @@ -190,6 +196,13 @@ namespace xt } } + template + xchunked_array(const xexpression& e) + { + const auto& chunk_shape = detail::chunk_helper::chunk_shape(e); + *this = xchunked_array(e, chunk_shape); + } + template self_type& operator=(const xexpression& e) { @@ -229,6 +242,11 @@ namespace xt return shape().size(); } + shape_type chunk_shape() const + { + return m_chunk_shape; + } + template bool broadcast_shape(S& s, bool reuse_cache = false) const { @@ -361,4 +379,3 @@ namespace xt } #endif - diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 507b26aed..15c3f1ae3 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -44,6 +44,9 @@ namespace xt TEST(xchunked_array, assign_expression) { +#ifdef _MSC_FULL_VER + std::cout << "MSC_FULL_VER = " << _MSC_FULL_VER << std::endl; +#endif std::vector shape1 = {2, 2, 2}; std::vector chunk_shape1 = {2, 3, 4}; chunked_array a1(shape1, chunk_shape1); @@ -75,13 +78,33 @@ namespace xt {{1., 2., 3.}, {4., 5., 6.}, {7., 8., 9.}}; + + EXPECT_EQ(xt::is_chunked(a3), false); + std::vector chunk_shape4 = {2, 2}; auto a4 = chunked_array(a3, chunk_shape4); + + EXPECT_EQ(xt::is_chunked(a4), true); + double i = 1.; for (const auto& v: a4) { EXPECT_EQ(v, i); i += 1.; } + + auto a5 = chunked_array(a4); + EXPECT_EQ(xt::is_chunked(a5), true); + for (const auto& v: a5.chunk_shape()) + { + EXPECT_EQ(v, 2); + } + + auto a6 = chunked_array(a3); + EXPECT_EQ(xt::is_chunked(a6), true); + for (const auto& v: a6.chunk_shape()) + { + EXPECT_EQ(v, 3); + } } } From dd3ec144cf257e2fcae3225d633d98e3029a181a Mon Sep 17 00:00:00 2001 From: gouarin Date: Fri, 17 Jul 2020 07:42:21 +0200 Subject: [PATCH 059/606] fix typo in the documentation --- docs/source/closure-semantics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/closure-semantics.rst b/docs/source/closure-semantics.rst index 301df73a6..47ecf850f 100644 --- a/docs/source/closure-semantics.rst +++ b/docs/source/closure-semantics.rst @@ -184,7 +184,7 @@ hold a const reference or a value for ``e`` depending on the lvalue-ness of the Reusing expressions / sharing expressions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Sometimes it is necessary to use a xexpression in two seperate places in another xexpression. For example, when computing +Sometimes it is necessary to use a xexpression in two separate places in another xexpression. For example, when computing something like ``sin(A) + cos(A)`` we can see A being referenced twice. This works fine if we can guarantee that ``A`` has a long enough lifetime. However, when writing generic interfaces that accept rvalues we cannot always guarantee that ``A`` will live long enough. @@ -230,7 +230,7 @@ But under certain circumstances it might be required, e.g. to implement a fully We can see that, before returning from the function, four copies of ``shared_weights`` exist: two in the two ``xt::sum`` functions, and one is the temporary. The last one lies -in ``weights`` itself, it is a technical requirement for the ``share`` syyntax. After +in ``weights`` itself, it is a technical requirement for the ``share`` syntax. After returning from the function, only two copies of the ``xshared_expression`` will exist. As discussed before, ``xt::make_xshared`` has the same overhead as creating a ``std::shared_ptr`` which is used internally by the shared expression. From ba9613a39c7631ca73e88ec3354f76fa13177f05 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 17 Jul 2020 13:57:02 +0200 Subject: [PATCH 060/606] Replace template parameter chunk_type with chunk_storage --- include/xtensor/xchunked_array.hpp | 49 ++++++++++++++++-------------- test/test_xchunked_array.cpp | 16 +++++----- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index ba8ada3d5..419c3a2fb 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -47,37 +47,40 @@ namespace xt return return_type::value; } - template + template class xchunked_array; - template - struct xcontainer_inner_types> + template + struct xcontainer_inner_types> { + using chunk_type = typename chunk_storage::value_type; using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; using size_type = std::size_t; using storage_type = chunk_type; - using temporary_type = xchunked_array; + using temporary_type = xchunked_array; }; - template - struct xiterable_inner_types> + template + struct xiterable_inner_types> { + using chunk_type = typename chunk_storage::value_type; using inner_shape_type = typename chunk_type::shape_type; - using const_stepper = xindexed_stepper, true>; - using stepper = xindexed_stepper, false>; + using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; }; - template - class xchunked_array: public xaccessible>, - public xiterable>, - public xcontainer_semantic> + template + class xchunked_array: public xaccessible>, + public xiterable>, + public xcontainer_semantic> { public: + using chunk_type = typename chunk_storage::value_type; using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; - using self_type = xchunked_array; + using self_type = xchunked_array; using semantic_base = xcontainer_semantic; using iterable_base = xconst_iterable; using const_stepper = typename iterable_base::const_stepper; @@ -200,7 +203,7 @@ namespace xt xchunked_array(const xexpression& e) { const auto& chunk_shape = detail::chunk_helper::chunk_shape(e); - *this = xchunked_array(e, chunk_shape); + *this = xchunked_array(e, chunk_shape); } template @@ -255,7 +258,7 @@ namespace xt private: - xarray m_chunks; + chunk_storage m_chunks; shape_type m_shape; shape_type m_chunk_shape; @@ -345,33 +348,33 @@ namespace xt } }; - template + template template - inline auto xchunked_array::stepper_begin(const O& shape) const noexcept -> const_stepper + inline auto xchunked_array::stepper_begin(const O& shape) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); return const_stepper(this, offset); } - template + template template - inline auto xchunked_array::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper + inline auto xchunked_array::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); return const_stepper(this, offset, true); } - template + template template - inline auto xchunked_array::stepper_begin(const O& shape) noexcept -> stepper + inline auto xchunked_array::stepper_begin(const O& shape) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); return stepper(this, offset); } - template + template template - inline auto xchunked_array::stepper_end(const O& shape, layout_type) noexcept -> stepper + inline auto xchunked_array::stepper_end(const O& shape, layout_type) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); return stepper(this, offset, true); diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 15c3f1ae3..c0756e5ed 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -14,7 +14,7 @@ namespace xt { - using chunked_array = xt::xchunked_array>; + using chunked_array = xchunked_array>>; TEST(xchunked_array, indexed_access) { @@ -53,7 +53,7 @@ namespace xt double val; val = 3.; - a1 = xt::broadcast(val, a1.shape()); + a1 = broadcast(val, a1.shape()); for (const auto& v: a1) { EXPECT_EQ(v, val); @@ -62,7 +62,7 @@ namespace xt std::vector shape2 = {32, 10, 10}; chunked_array a2(shape2, chunk_shape1); - a2 = xt::broadcast(val, a2.shape()); + a2 = broadcast(val, a2.shape()); for (const auto& v: a2) { EXPECT_EQ(v, val); @@ -74,17 +74,17 @@ namespace xt EXPECT_EQ(v, 2. * val); } - xt::xarray a3 + xarray a3 {{1., 2., 3.}, {4., 5., 6.}, {7., 8., 9.}}; - EXPECT_EQ(xt::is_chunked(a3), false); + EXPECT_EQ(is_chunked(a3), false); std::vector chunk_shape4 = {2, 2}; auto a4 = chunked_array(a3, chunk_shape4); - EXPECT_EQ(xt::is_chunked(a4), true); + EXPECT_EQ(is_chunked(a4), true); double i = 1.; for (const auto& v: a4) @@ -94,14 +94,14 @@ namespace xt } auto a5 = chunked_array(a4); - EXPECT_EQ(xt::is_chunked(a5), true); + EXPECT_EQ(is_chunked(a5), true); for (const auto& v: a5.chunk_shape()) { EXPECT_EQ(v, 2); } auto a6 = chunked_array(a3); - EXPECT_EQ(xt::is_chunked(a6), true); + EXPECT_EQ(is_chunked(a6), true); for (const auto& v: a6.chunk_shape()) { EXPECT_EQ(v, 3); From d44d8ae9a0b2b742fdc0991760c2d7a6288f6780 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 24 Jul 2020 10:43:28 +0200 Subject: [PATCH 061/606] Implement on-disk chunked array (#2096) First mmplementation of on-disk chunked array --- include/xtensor/xchunk_store_manager.hpp | 194 +++++++++++++++++++++++ include/xtensor/xdisk_io_handler.hpp | 48 ++++++ include/xtensor/xfile_array.hpp | 163 +++++++++++++++++++ test/test_xchunked_array.cpp | 30 ++++ 4 files changed, 435 insertions(+) create mode 100644 include/xtensor/xchunk_store_manager.hpp create mode 100644 include/xtensor/xdisk_io_handler.hpp create mode 100644 include/xtensor/xfile_array.hpp diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp new file mode 100644 index 000000000..5b4699c1c --- /dev/null +++ b/include/xtensor/xchunk_store_manager.hpp @@ -0,0 +1,194 @@ +#ifndef XTENSOR_CHUNK_STORE_MANAGER_HPP +#define XTENSOR_CHUNK_STORE_MANAGER_HPP + +#include +#include +#include +#include +#include + +#include "xarray.hpp" +#include "xcsv.hpp" +#include "xio.hpp" + +namespace xt +{ + template + class xchunk_store_manager; + + template + struct xcontainer_inner_types> + { + using const_reference = const EC&; + using reference = EC&; + using size_type = std::size_t; + using storage_type = EC; + using temporary_type = xchunk_store_manager; + }; + + template + struct xiterable_inner_types> + { + using inner_shape_type = std::vector; + using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; + }; + + template + class xchunk_store_manager: public xaccessible>, + public xiterable> + { + public: + + using const_reference = const EC&; + using reference = EC&; + using self_type = xchunk_store_manager; + using iterable_base = xconst_iterable; + using const_stepper = typename iterable_base::const_stepper; + using stepper = typename iterable_base::stepper; + using inner_types = xcontainer_inner_types; + using size_type = typename inner_types::size_type; + using storage_type = typename inner_types::storage_type; + using value_type = storage_type; + using pointer = value_type*; + using const_pointer = const value_type*; + using difference_type = std::ptrdiff_t; + using shape_type = std::vector; + + template + const_stepper stepper_begin(const O& shape) const noexcept; + template + const_stepper stepper_end(const O& shape, layout_type) const noexcept; + + template + stepper stepper_begin(const O& shape) noexcept; + template + stepper stepper_end(const O& shape, layout_type) noexcept; + + const shape_type& shape() const + { + return m_shape; + } + + xchunk_store_manager() + { + } + + template + void map_file_array(I first, I last) + { + std::string path; + for (auto it = first; it != last; ++it) + { + if (!path.empty()) + path.append("."); + path.append(std::to_string(*it)); + } + m_file_array.set_path(path); + } + + template + inline const_reference operator()(Idxs... idxs) const + { + auto index = get_indexes(idxs...); + map_file_array(index.cbegin(), index.cend()); + return m_file_array; + } + + template + inline reference operator()(Idxs... idxs) + { + auto index = get_indexes(idxs...); + map_file_array(index.cbegin(), index.cend()); + return m_file_array; + } + + xchunk_store_manager(const xchunk_store_manager&) = default; + xchunk_store_manager& operator=(const xchunk_store_manager&) = default; + + xchunk_store_manager(xchunk_store_manager&&) = default; + xchunk_store_manager& operator=(xchunk_store_manager&&) = default; + + reference operator[](const xindex& index) + { + map_file_array(index.cbegin(), index.cend()); + return m_file_array; + } + + const_reference operator[](const xindex& index) const + { + map_file_array(index.cbegin(), index.cend()); + return m_file_array; + } + + template + inline reference element(It first, It last) + { + map_file_array(first, last); + return m_file_array; + } + + template + inline const_reference element(It first, It last) const + { + map_file_array(first, last); + return m_file_array; + } + + size_type dimension() const + { + return shape().size(); + } + + template + void resize(S& shape) + { + } + + private: + + shape_type m_shape; + EC m_file_array; + + template + inline std::array get_indexes(Idxs... idxs) const + { + std::array indexes = {{idxs...}}; + return indexes; + } + }; + + template + template + inline auto xchunk_store_manager::stepper_begin(const O& shape) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset); + } + + template + template + inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset, true); + } + + template + template + inline auto xchunk_store_manager::stepper_begin(const O& shape) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset); + } + + template + template + inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset, true); + } +} + +#endif diff --git a/include/xtensor/xdisk_io_handler.hpp b/include/xtensor/xdisk_io_handler.hpp new file mode 100644 index 000000000..54a78bb98 --- /dev/null +++ b/include/xtensor/xdisk_io_handler.hpp @@ -0,0 +1,48 @@ +#ifndef XTENSOR_DISK_IO_HANDLER_HPP +#define XTENSOR_DISK_IO_HANDLER_HPP + +#include "xarray.hpp" +#include "xcsv.hpp" + +namespace xt +{ + template + class xdisk_io_handler + { + public: + + void set_array(xarray& array) + { + m_array = &array; + } + + void write(std::string& path) + { + if (m_out_file.is_open()) + m_out_file.close(); + m_out_file.open(path); + if (m_out_file.is_open()) + dump_csv(m_out_file, *m_array); + } + + void read(std::string& path) + { + if (m_in_file.is_open()) + m_in_file.close(); + m_in_file.open(path); + if (m_in_file.is_open()) + *m_array = load_csv(m_in_file); + else + *m_array = broadcast(0, m_array->shape()); + } + + private: + + std::ifstream m_in_file; + std::ofstream m_out_file; + xarray* m_array; + + }; +} + +#endif diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp new file mode 100644 index 000000000..c1f2564ff --- /dev/null +++ b/include/xtensor/xfile_array.hpp @@ -0,0 +1,163 @@ +#ifndef XTENSOR_FILE_ARRAY_HPP +#define XTENSOR_FILE_ARRAY_HPP + +#include "xarray.hpp" + +namespace xt +{ + template + class xfile_reference + { + public: + + xfile_reference(EC& value, bool& array_dirty) + { + m_pvalue = &value; + m_parray_dirty = &array_dirty; + } + + operator const EC() const + { + return *m_pvalue; + } + + operator EC() + { + return *m_pvalue; + } + + EC operator=(const EC value) + { + if (value != *m_pvalue) + { + *m_parray_dirty = true; + *m_pvalue = value; + } + return *m_pvalue; + } + + private: + + EC* m_pvalue; + bool* m_parray_dirty; + + }; + + template + class xfile_array; + + template + struct xcontainer_inner_types> + { + using const_reference = const EC&; + using reference = EC&; + using size_type = std::size_t; + using storage_type = EC; + }; + + template + struct xiterable_inner_types> + { + using inner_shape_type = std::vector; + using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; + }; + + template + class xfile_array: public xaccessible>, + public xiterable> + { + public: + + using const_reference = const EC&; + using reference = xfile_reference; + using shape_type = std::vector; + using self_type = xfile_array; + using inner_types = xcontainer_inner_types; + using size_type = typename inner_types::size_type; + using storage_type = typename inner_types::storage_type; + using value_type = storage_type; + + ~xfile_array() + { + if (m_array_dirty) + m_io_handler.write(m_path); + } + + void set_path(std::string& path) + { + if (path != m_path) + { + // maybe write to old file + if (m_array_dirty) + { + m_io_handler.write(m_path); + m_array_dirty = false; + } + m_path = path; + // read new file + m_io_handler.read(path); + } + } + + template + void resize(S& shape) + { + m_array.resize(shape); + m_array = broadcast(0, shape); + m_io_handler.set_array(m_array); + } + + template + inline reference operator()(Idxs... idxs) + { + auto index = get_indexes(idxs...); + return reference(m_array.element(index.cbegin(), index.cend()), m_array_dirty); + } + + template + inline const_reference operator()(Idxs... idxs) const + { + auto index = get_indexes(idxs...); + return m_array.element(index.cbegin(), index.cend()); + } + + reference operator[](const xindex& index) + { + return reference(m_array.element(index.cbegin(), index.cend()), m_array_dirty); + } + + const_reference operator[](const xindex& index) const + { + return m_array.element(index.cbegin(), index.cend()); + } + + template + inline reference element(It first, It last) + { + return reference(m_array.element(first, last), m_array_dirty); + } + + template + inline const_reference element(It first, It last) const + { + return m_array.element(first, last); + } + + private: + + xarray m_array; + bool m_array_dirty; + io_handler m_io_handler; + std::string m_path; + + template + inline std::array get_indexes(Idxs... idxs) const + { + std::array indexes = {{idxs...}}; + return indexes; + } + }; +} + +#endif diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index c0756e5ed..6474aafc5 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -11,6 +11,9 @@ #include "xtensor/xbroadcast.hpp" #include "xtensor/xchunked_array.hpp" +#include "xtensor/xchunk_store_manager.hpp" +#include "xtensor/xfile_array.hpp" +#include "xtensor/xdisk_io_handler.hpp" namespace xt { @@ -107,4 +110,31 @@ namespace xt EXPECT_EQ(v, 3); } } + + TEST(xchunked_array, disk_array) + { + std::vector shape = {4, 4}; + std::vector chunk_shape = {2, 2}; + xchunked_array>>> a1(shape, chunk_shape); + std::vector idx = {1, 2}; + double v1 = 3.4; + double v2 = 5.6; + a1(2, 1) = v1; + a1[idx] = v2; + ASSERT_EQ(a1(2, 1), v1); + ASSERT_EQ(a1[idx], v2); + + std::ifstream in_file; + in_file.open("0.1"); + auto data = xt::load_csv(in_file); + xt::xarray ref = {{0, 0}, {v2, 0}}; + EXPECT_EQ(data, ref); + in_file.close(); + + in_file.open("1.0"); + data = xt::load_csv(in_file); + ref = {{0, v1}, {0, 0}}; + EXPECT_EQ(data, ref); + in_file.close(); + } } From 65e2c2ef115b4014ed198ca6f96052c498068b50 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 11 Aug 2020 11:52:07 +0200 Subject: [PATCH 062/606] Fixed CI --- .appveyor.yml | 5 ++--- .azure-pipelines/azure-pipelines-win.yml | 3 +-- test/CMakeLists.txt | 4 +++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 39c064ab2..378162cb7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -33,12 +33,11 @@ install: - conda info -a - conda env create --file environment-dev.yml - CALL conda.bat activate xtensor - - conda install gtest=1.10.0 -c conda-forge - ps: if($env:APPVEYOR_BUILD_WORKER_IMAGE -eq "Visual Studio 2015") { - cmake -G "NMake Makefiles" -DCMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DBUILD_TESTS=ON -DXTENSOR_USE_XSIMD=ON -DCMAKE_BUILD_TYPE=RELEASE . + cmake -G "NMake Makefiles" -DCMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DDOWNLOAD_GTEST=ON -DXTENSOR_USE_XSIMD=ON -DCMAKE_BUILD_TYPE=RELEASE . } else { - cmake -G "NMake Makefiles" -D CMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DBUILD_TESTS=ON -DDISABLE_VS2017=ON -DCMAKE_BUILD_TYPE=RELEASE . + cmake -G "NMake Makefiles" -DCMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DDOWNLOAD_GTEST=ON -DDISABLE_VS2017=ON -DCMAKE_BUILD_TYPE=RELEASE . } - nmake test_xtensor_lib - cd test diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index de8af69b1..1a00afdc8 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -35,7 +35,6 @@ jobs: # Note: conda activate doesn't work here, because it creates a new shell! - script: | conda install cmake==3.14.0 ^ - gtest==1.10.0 ^ ninja ^ nlohmann_json ^ xtl==0.6.12 ^ @@ -62,7 +61,7 @@ jobs: -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_C_COMPILER=clang-cl ^ -DCMAKE_CXX_COMPILER=clang-cl ^ - -DBUILD_TESTS=ON ^ + -DDOWNLOAD_GTEST=ON ^ -DXTENSOR_USE_XSIMD=ON ^ $(Build.SourcesDirectory) displayName: "Configure xtensor" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7ed9a876a..57d935c8b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -92,7 +92,7 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions") endif() else() # We are using clang-cl - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_cxx_std_flag} /MP /bigobj") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_cxx_std_flag} /bigobj") set(CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO) add_definitions(-D_CRT_SECURE_NO_WARNINGS) add_definitions(-D_SILENCE_TR1_NAMESPACE_DEPRECATION_WARNING) @@ -127,6 +127,8 @@ if(DOWNLOAD_GTEST OR GTEST_SRC_DIR) message(FATAL_ERROR "Build step for googletest failed: ${result}") endif() + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + # Add googletest directly to our build. This defines # the gtest and gtest_main targets. add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src From 9f5c7a44e97610d4909aed0b8201cb12d89e2ff9 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 11 Aug 2020 21:44:38 +0200 Subject: [PATCH 063/606] to squash --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5e3e4d30e..84d8f4c1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -114,7 +114,7 @@ matrix: env: COMPILER=gcc GCC=9 ENABLE_CPP20=1 - os: osx if: branch = master - osx_image: xcode8 + osx_image: xcode9 compiler: clang allow_failures: - env: COMPILER=gcc GCC=9 ENABLE_CPP20=1 From ab44bb17ec5f494c8005a52250d06f7290b42b73 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 11 Aug 2020 21:44:45 +0200 Subject: [PATCH 064/606] testing appveyor --- .appveyor.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 378162cb7..e96c2d10b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -33,12 +33,9 @@ install: - conda info -a - conda env create --file environment-dev.yml - CALL conda.bat activate xtensor - - ps: if($env:APPVEYOR_BUILD_WORKER_IMAGE -eq "Visual Studio 2015") { - cmake -G "NMake Makefiles" -DCMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DDOWNLOAD_GTEST=ON -DXTENSOR_USE_XSIMD=ON -DCMAKE_BUILD_TYPE=RELEASE . - } - else { - cmake -G "NMake Makefiles" -DCMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DDOWNLOAD_GTEST=ON -DDISABLE_VS2017=ON -DCMAKE_BUILD_TYPE=RELEASE . - } + - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" set CMAKE_ARGS="-DDISABLE_VS2017=ON" + - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" set CMAKE_ARGS="" + - cmake -G "NMake Makefiles" -DCMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DDOWNLOAD_GTEST=ON -DXTENSOR_USE_XSIMD=ON -DCMAKE_BUILD_TYPE=RELEASE %CMAKE_ARGS% . - nmake test_xtensor_lib - cd test From ff36a7220d68278ee1cc520923ffb1386649bcc9 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 11 Aug 2020 23:57:09 +0200 Subject: [PATCH 065/606] Implement chunk pool in xchunk_store_manager (#2099) Implement chunk pool in xchunk_store_manager --- include/xtensor/xchunk_store_manager.hpp | 101 ++++++++++++++++++----- include/xtensor/xchunked_array.hpp | 5 ++ include/xtensor/xdisk_io_handler.hpp | 14 ++-- include/xtensor/xfile_array.hpp | 13 +++ test/test_xchunked_array.cpp | 21 ++++- 5 files changed, 124 insertions(+), 30 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 5b4699c1c..b054d2225 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -29,14 +29,14 @@ namespace xt template struct xiterable_inner_types> { - using inner_shape_type = std::vector; + using inner_shape_type = std::vector; using const_stepper = xindexed_stepper, true>; using stepper = xindexed_stepper, false>; }; template class xchunk_store_manager: public xaccessible>, - public xiterable> + public xiterable> { public: @@ -53,7 +53,7 @@ namespace xt using pointer = value_type*; using const_pointer = const value_type*; using difference_type = std::ptrdiff_t; - using shape_type = std::vector; + using shape_type = std::vector; template const_stepper stepper_begin(const O& shape) const noexcept; @@ -72,35 +72,96 @@ namespace xt xchunk_store_manager() { + // default pool size is 1 + // so that first chunk is always resized to the chunk shape + m_chunk_pool.resize(1); + m_index_pool.resize(1); + m_unload_index = 0; + } + + void set_pool_size(std::size_t n) + { + // first chunk always has the correct shape + // get the shape before resizing the pool + auto chunk_shape = m_chunk_pool[0].array().shape(); + m_chunk_pool.resize(n); + m_index_pool.resize(n); + m_unload_index = 0; + // resize the pool chunks + for (auto& chunk: m_chunk_pool) + { + chunk.resize(chunk_shape); + } + } + + void flush() + { + for (auto& chunk: m_chunk_pool) + { + chunk.flush(); + } } template - void map_file_array(I first, I last) + EC& map_file_array(I first, I last) { std::string path; + std::vector index; for (auto it = first; it != last; ++it) { if (!path.empty()) + { path.append("."); + } path.append(std::to_string(*it)); + index.push_back(*it); + } + if (index.empty()) + { + return m_chunk_pool[0]; + } + else + { + // check if the chunk is already loaded in memory + const auto it1 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), index); + std::size_t i; + if (it1 != m_index_pool.cend()) + { + i = std::distance(m_index_pool.cbegin(), it1); + return m_chunk_pool[i]; + } + // if not, find a free chunk in the pool + std::vector empty_index; + const auto it2 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), empty_index); + if (it2 != m_index_pool.cend()) + { + i = std::distance(m_index_pool.cbegin(), it2); + m_chunk_pool[i].set_path(path); + m_index_pool[i] = index; + return m_chunk_pool[i]; + } + // no free chunk, take one (which will thus be unloaded) + // fairness is guaranteed through the use of a walking index + m_chunk_pool[m_unload_index].set_path(path); + m_index_pool[m_unload_index] = index; + auto& chunk = m_chunk_pool[m_unload_index]; + m_unload_index = (m_unload_index + 1) % m_index_pool.size(); + return chunk; } - m_file_array.set_path(path); } template inline const_reference operator()(Idxs... idxs) const { auto index = get_indexes(idxs...); - map_file_array(index.cbegin(), index.cend()); - return m_file_array; + return map_file_array(index.cbegin(), index.cend()); } template inline reference operator()(Idxs... idxs) { auto index = get_indexes(idxs...); - map_file_array(index.cbegin(), index.cend()); - return m_file_array; + return map_file_array(index.cbegin(), index.cend()); } xchunk_store_manager(const xchunk_store_manager&) = default; @@ -111,28 +172,24 @@ namespace xt reference operator[](const xindex& index) { - map_file_array(index.cbegin(), index.cend()); - return m_file_array; + return map_file_array(index.cbegin(), index.cend()); } const_reference operator[](const xindex& index) const { - map_file_array(index.cbegin(), index.cend()); - return m_file_array; + return map_file_array(index.cbegin(), index.cend()); } template inline reference element(It first, It last) { - map_file_array(first, last); - return m_file_array; + return map_file_array(first, last); } template inline const_reference element(It first, It last) const { - map_file_array(first, last); - return m_file_array; + return map_file_array(first, last); } size_type dimension() const @@ -143,17 +200,21 @@ namespace xt template void resize(S& shape) { + // don't resize according to total number of chunks + // instead the pool manages a number of in-memory chunks } private: shape_type m_shape; - EC m_file_array; + std::vector m_chunk_pool; + std::vector> m_index_pool; + std::size_t m_unload_index; template - inline std::array get_indexes(Idxs... idxs) const + inline std::array get_indexes(Idxs... idxs) const { - std::array indexes = {{idxs...}}; + std::array indexes = {{idxs...}}; return indexes; } }; diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 419c3a2fb..d832324c8 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -199,6 +199,11 @@ namespace xt } } + chunk_storage& chunks() + { + return m_chunks; + } + template xchunked_array(const xexpression& e) { diff --git a/include/xtensor/xdisk_io_handler.hpp b/include/xtensor/xdisk_io_handler.hpp index 54a78bb98..47be3acaa 100644 --- a/include/xtensor/xdisk_io_handler.hpp +++ b/include/xtensor/xdisk_io_handler.hpp @@ -18,28 +18,30 @@ namespace xt void write(std::string& path) { - if (m_out_file.is_open()) - m_out_file.close(); + std::ofstream m_out_file; m_out_file.open(path); if (m_out_file.is_open()) + { dump_csv(m_out_file, *m_array); + m_out_file.close(); + } } void read(std::string& path) { - if (m_in_file.is_open()) - m_in_file.close(); + std::ifstream m_in_file; m_in_file.open(path); if (m_in_file.is_open()) + { *m_array = load_csv(m_in_file); + m_in_file.close(); + } else *m_array = broadcast(0, m_array->shape()); } private: - std::ifstream m_in_file; - std::ofstream m_out_file; xarray* m_array; }; diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index c1f2564ff..70e4e18c9 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -79,9 +79,22 @@ namespace xt using value_type = storage_type; ~xfile_array() + { + flush(); + } + + void flush() { if (m_array_dirty) + { m_io_handler.write(m_path); + m_array_dirty = false; + } + } + + xarray& array() + { + return m_array; } void set_path(std::string& path) diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 6474aafc5..8c506d377 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -116,24 +116,37 @@ namespace xt std::vector shape = {4, 4}; std::vector chunk_shape = {2, 2}; xchunked_array>>> a1(shape, chunk_shape); + a1.chunks().set_pool_size(2); std::vector idx = {1, 2}; double v1 = 3.4; double v2 = 5.6; + double v3 = 7.8; a1(2, 1) = v1; a1[idx] = v2; + a1(0, 0) = v3; // this should unload chunk 1.0 ASSERT_EQ(a1(2, 1), v1); ASSERT_EQ(a1[idx], v2); + ASSERT_EQ(a1(0, 0), v3); std::ifstream in_file; + xt::xarray ref; + xt::xarray data; + in_file.open("1.0"); + data = xt::load_csv(in_file); + ref = {{0, v1}, {0, 0}}; + EXPECT_EQ(data, ref); + in_file.close(); + + a1.chunks().flush(); in_file.open("0.1"); - auto data = xt::load_csv(in_file); - xt::xarray ref = {{0, 0}, {v2, 0}}; + data = xt::load_csv(in_file); + ref = {{0, 0}, {v2, 0}}; EXPECT_EQ(data, ref); in_file.close(); - in_file.open("1.0"); + in_file.open("0.0"); data = xt::load_csv(in_file); - ref = {{0, v1}, {0, 0}}; + ref = {{v3, 0}, {0, 0}}; EXPECT_EQ(data, ref); in_file.close(); } From 5195fdd72119b2456a347dc17bd5cba9b98e68f9 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 27 Jul 2020 18:03:38 +0200 Subject: [PATCH 066/606] Make xfile_array an expression --- include/xtensor/xchunk_store_manager.hpp | 3 - include/xtensor/xdisk_io_handler.hpp | 22 +-- include/xtensor/xfile_array.hpp | 190 +++++++++++++++++++++-- test/test_xchunked_array.cpp | 49 +++++- 4 files changed, 235 insertions(+), 29 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index b054d2225..d33bc90f8 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -1,9 +1,6 @@ #ifndef XTENSOR_CHUNK_STORE_MANAGER_HPP #define XTENSOR_CHUNK_STORE_MANAGER_HPP -#include -#include -#include #include #include diff --git a/include/xtensor/xdisk_io_handler.hpp b/include/xtensor/xdisk_io_handler.hpp index 47be3acaa..aff422ad4 100644 --- a/include/xtensor/xdisk_io_handler.hpp +++ b/include/xtensor/xdisk_io_handler.hpp @@ -11,39 +11,31 @@ namespace xt { public: - void set_array(xarray& array) - { - m_array = &array; - } - - void write(std::string& path) + void write(xarray& array, std::string& path) { std::ofstream m_out_file; m_out_file.open(path); if (m_out_file.is_open()) { - dump_csv(m_out_file, *m_array); + dump_csv(m_out_file, array); m_out_file.close(); } } - void read(std::string& path) + void read(xarray& array, std::string& path) { std::ifstream m_in_file; m_in_file.open(path); if (m_in_file.is_open()) { - *m_array = load_csv(m_in_file); + array = load_csv(m_in_file); m_in_file.close(); } else - *m_array = broadcast(0, m_array->shape()); + { + array = broadcast(0, array.shape()); + } } - - private: - - xarray* m_array; - }; } diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index 70e4e18c9..83c2c3fbe 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -1,10 +1,52 @@ #ifndef XTENSOR_FILE_ARRAY_HPP #define XTENSOR_FILE_ARRAY_HPP +#include +#include +#include + #include "xarray.hpp" +#include "xnoalias.hpp" namespace xt { + namespace detail + { + // Workaround for VS2015 + template + using try_path = decltype(std::declval().path()); + + template class OP, class = void> + struct file_helper_impl + { + static const char* path(const xexpression& e) + { + return ""; + } + using is_stored = std::false_type; + }; + + template class OP> + struct file_helper_impl>> + { + static const char* path(const xexpression& e) + { + return e.derived_cast().path(); + } + using is_stored = std::true_type; + }; + + template + using file_helper = file_helper_impl; + } + + template + constexpr bool is_stored(const xexpression& e) + { + using return_type = typename detail::file_helper::is_stored; + return return_type::value; + } + template class xfile_reference { @@ -50,44 +92,83 @@ namespace xt struct xcontainer_inner_types> { using const_reference = const EC&; - using reference = EC&; + using reference = xfile_reference; using size_type = std::size_t; using storage_type = EC; + using temporary_type = xfile_array; }; template struct xiterable_inner_types> { - using inner_shape_type = std::vector; + using inner_shape_type = typename xarray::shape_type; using const_stepper = xindexed_stepper, true>; using stepper = xindexed_stepper, false>; }; template class xfile_array: public xaccessible>, - public xiterable> + public xiterable>, + public xcontainer_semantic> { public: using const_reference = const EC&; using reference = xfile_reference; - using shape_type = std::vector; using self_type = xfile_array; + using semantic_base = xcontainer_semantic; + using iterable_base = xconst_iterable; + using const_stepper = typename iterable_base::const_stepper; + using stepper = typename iterable_base::stepper; using inner_types = xcontainer_inner_types; using size_type = typename inner_types::size_type; using storage_type = typename inner_types::storage_type; using value_type = storage_type; + using pointer = value_type*; + using const_pointer = const value_type*; + using difference_type = std::ptrdiff_t; + using shape_type = typename xarray::shape_type; + using temporary_type = typename inner_types::temporary_type; + using bool_load_type = xt::bool_load_type; + static constexpr layout_type static_layout = layout_type::dynamic; + + template + const_stepper stepper_begin(const O& shape) const noexcept; + template + const_stepper stepper_end(const O& shape, layout_type) const noexcept; + + template + stepper stepper_begin(const O& shape) noexcept; + template + stepper stepper_end(const O& shape, layout_type) noexcept; + + const auto& shape() const + { + return m_array.shape(); + } + + inline layout_type layout() const noexcept + { + return static_layout; + } + + inline bool is_contiguous() const noexcept + { + return false; + } + + xfile_array() {} ~xfile_array() { flush(); } - + void flush() { if (m_array_dirty) { - m_io_handler.write(m_path); + m_io_handler.write(m_array, m_path); m_array_dirty = false; } } @@ -97,6 +178,12 @@ namespace xt return m_array; } + void set_path(const char* path) + { + std::string p(path); + set_path(p); + } + void set_path(std::string& path) { if (path != m_path) @@ -104,12 +191,12 @@ namespace xt // maybe write to old file if (m_array_dirty) { - m_io_handler.write(m_path); + m_io_handler.write(m_array, m_path); m_array_dirty = false; } m_path = path; // read new file - m_io_handler.read(path); + m_io_handler.read(m_array, path); } } @@ -118,7 +205,6 @@ namespace xt { m_array.resize(shape); m_array = broadcast(0, shape); - m_io_handler.set_array(m_array); } template @@ -135,6 +221,38 @@ namespace xt return m_array.element(index.cbegin(), index.cend()); } + xfile_array(const xfile_array&) = default; + xfile_array& operator=(const xfile_array&) = default; + + xfile_array(xfile_array&&) = default; + xfile_array& operator=(xfile_array&&) = default; + + template + xfile_array(const xexpression& e, const char* path) + { + set_path(path); + m_array_dirty = true; + const auto& shape = e.derived_cast().shape(); + m_array.resize(shape); + xstrided_slice_vector sv; + for (auto i = 0; i < dimension(); i++) + sv.push_back(all()); + noalias(m_array) = strided_view(e.derived_cast(), sv); + } + + template + xfile_array(const xexpression& e) + { + const char* path = detail::file_helper::path(e); + *this = xfile_array(e, path); + } + + template + self_type& operator=(const xexpression& e) + { + return semantic_base::operator=(e); + } + reference operator[](const xindex& index) { return reference(m_array.element(index.cbegin(), index.cend()), m_array_dirty); @@ -157,6 +275,22 @@ namespace xt return m_array.element(first, last); } + size_type dimension() const + { + return shape().size(); + } + + const char* path() const + { + return m_path.c_str(); + } + + template + bool broadcast_shape(S& s, bool reuse_cache = false) const + { + return xt::broadcast_shape(shape(), s); + } + private: xarray m_array; @@ -170,7 +304,45 @@ namespace xt std::array indexes = {{idxs...}}; return indexes; } + + template + bool is_trivial_broadcast(const S& str) const noexcept + { + return false; + } }; + + template + template + inline auto xfile_array::stepper_begin(const O& shape) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset); + } + + template + template + inline auto xfile_array::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset, true); + } + + template + template + inline auto xfile_array::stepper_begin(const O& shape) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset); + } + + template + template + inline auto xfile_array::stepper_end(const O& shape, layout_type) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset, true); + } } #endif diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 8c506d377..492962c88 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -115,7 +115,7 @@ namespace xt { std::vector shape = {4, 4}; std::vector chunk_shape = {2, 2}; - xchunked_array>>> a1(shape, chunk_shape); + xchunked_array>>> a1(shape, chunk_shape); a1.chunks().set_pool_size(2); std::vector idx = {1, 2}; double v1 = 3.4; @@ -136,7 +136,7 @@ namespace xt ref = {{0, v1}, {0, 0}}; EXPECT_EQ(data, ref); in_file.close(); - + a1.chunks().flush(); in_file.open("0.1"); data = xt::load_csv(in_file); @@ -150,4 +150,49 @@ namespace xt EXPECT_EQ(data, ref); in_file.close(); } + + TEST(xfile_array, indexed_access) + { + std::vector shape = {2, 2, 2}; + xfile_array> a; + a.resize(shape); + double val = 3.; + for (auto it: a) + it = val; + for (auto it: a) + ASSERT_EQ(it, val); + } + + TEST(xfile_array, assign_expression) + { + double v1 = 3.; + auto a1 = xfile_array>(broadcast(v1, {2, 2}), "a1"); + for (const auto& v: a1) + { + EXPECT_EQ(v, v1); + } + + double v2 = 2. * v1; + auto a2 = xfile_array>(a1 + a1, "a2"); + for (const auto& v: a2) + { + EXPECT_EQ(v, v2); + } + + a1.flush(); + a2.flush(); + + std::ifstream in_file; + in_file.open("a1"); + auto data = load_csv(in_file); + xarray ref = {{v1, v1}, {v1, v1}}; + EXPECT_EQ(data, ref); + in_file.close(); + + in_file.open("a2"); + data = load_csv(in_file); + ref = {{v2, v2}, {v2, v2}}; + EXPECT_EQ(data, ref); + in_file.close(); + } } From 0481503bd64f7ab131535d2c2b6bf32f180f1922 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 14 Aug 2020 11:41:51 +0200 Subject: [PATCH 067/606] xchunked_array code cleanup --- include/xtensor/xchunked_array.hpp | 604 ++++++++++++++++------------- 1 file changed, 344 insertions(+), 260 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index d832324c8..971a1c50e 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -10,42 +10,10 @@ namespace xt { - namespace detail - { - // Workaround for VS2015 - template - using try_chunk_shape = decltype(std::declval().chunk_shape()); - template class OP, class = void> - struct chunk_helper_impl - { - static const auto& chunk_shape(const xexpression& e) - { - return e.derived_cast().shape(); - } - using is_chunked = std::false_type; - }; - - template class OP> - struct chunk_helper_impl>> - { - static const auto& chunk_shape(const xexpression& e) - { - return e.derived_cast().chunk_shape(); - } - using is_chunked = std::true_type; - }; - - template - using chunk_helper = chunk_helper_impl; - } - - template - constexpr bool is_chunked(const xexpression& e) - { - using return_type = typename detail::chunk_helper::is_chunked; - return return_type::value; - } + /****************************** + * xchunked_array declaration * + ******************************/ template class xchunked_array; @@ -77,6 +45,7 @@ namespace xt { public: + using chunk_storage_type = chunk_storage; using chunk_type = typename chunk_storage::value_type; using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; @@ -97,292 +66,407 @@ namespace xt using bool_load_type = xt::bool_load_type; static constexpr layout_type static_layout = layout_type::dynamic; - template - const_stepper stepper_begin(const O& shape) const noexcept; - template - const_stepper stepper_end(const O& shape, layout_type) const noexcept; + template + xchunked_array(S&& shape, S&& chunk_shape); - template - stepper stepper_begin(const O& shape) noexcept; - template - stepper stepper_end(const O& shape, layout_type) noexcept; + xchunked_array(const xchunked_array&) = default; + xchunked_array& operator=(const xchunked_array&) = default; - const shape_type& shape() const - { - return m_shape; - } + xchunked_array(xchunked_array&&) = default; + xchunked_array& operator=(xchunked_array&&) = default; - inline layout_type layout() const noexcept - { - return static_layout; - } + template + xchunked_array(const xexpression& e); - inline bool is_contiguous() const noexcept - { - return false; - } + template + xchunked_array(const xexpression& e, S&& chunk_shape); + + template + xchunked_array& operator=(const xexpression& e); + + const shape_type& shape() const noexcept; + layout_type layout() const noexcept; + bool is_contiguous() const noexcept; template - inline const_reference operator()(Idxs... idxs) const - { - auto ii = get_indexes(idxs...); - auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); - return chunk.element(ii.second.cbegin(), ii.second.cend()); - } + reference operator()(Idxs... idxs); template - inline reference operator()(Idxs... idxs) - { - auto ii = get_indexes(idxs...); - auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); - return chunk.element(ii.second.cbegin(), ii.second.cend()); - } + const_reference operator()(Idxs... idxs) const; + + template + reference element(It first, It last); + + template + const_reference element(It first, It last) const; template - xchunked_array(S shape, S chunk_shape) - { - resize(shape, chunk_shape); - } + bool broadcast_shape(S& s, bool reuse_cache = false) const; - xchunked_array(const xchunked_array&) = default; - xchunked_array& operator=(const xchunked_array&) = default; + template + stepper stepper_begin(const S& shape) noexcept; + template + stepper stepper_end(const S& shape, layout_type) noexcept; - xchunked_array(xchunked_array&&) = default; - xchunked_array& operator=(xchunked_array&&) = default; + template + const_stepper stepper_begin(const S& shape) const noexcept; + template + const_stepper stepper_end(const S& shape, layout_type) const noexcept; - template - xchunked_array(const xexpression& e, S chunk_shape) + const shape_type& chunk_shape() const; + chunk_storage_type& chunks(); + const chunk_storage_type& chunks() const; + + private: + + template + using indexes_type = std::pair, std::array>; + + template + using chunk_indexes_type = std::array, sizeof...(Idxs)>; + + template + using static_indexes_type = std::pair, std::array>; + + using dynamic_indexes_type = std::pair, std::vector>; + + template + void resize(S1&& shape, S2&& chunk_shape); + + template + indexes_type get_indexes(Idxs... idxs) const; + + template + std::pair get_chunk_indexes_in_dimension(size_t dim, Idx idx) const; + + template + chunk_indexes_type get_chunk_indexes(std::index_sequence, Idxs... idxs) const; + + template + static_indexes_type unpack(const std::array &arr) const; + + template + dynamic_indexes_type get_indexes_dynamic(It first, It last) const; + + shape_type m_shape; + shape_type m_chunk_shape; + chunk_storage_type m_chunks; + }; + + template + constexpr bool is_chunked(const xexpression& e); + + /******************************* + * chunk_helper implementation * + *******************************/ + + namespace detail + { + // Workaround for VS2015 + template + using try_chunk_shape = decltype(std::declval().chunk_shape()); + + template class OP, class = void> + struct chunk_helper_impl { - const auto& shape = e.derived_cast().shape(); - resize_container(m_shape, shape.size()); - std::copy(shape.begin(), shape.end(), m_shape.begin()); - m_chunk_shape = chunk_shape; - resize(m_shape, m_chunk_shape); - shape_type ic(dimension()); // index of chunk, initialized to 0... - xstrided_slice_vector sv; // element slice corresponding to chunk - size_t di = 0; // dimension index - // initialize slice - for (auto i: ic) + using is_chunked = std::false_type; + static const auto& chunk_shape(const xexpression& e) { - sv.push_back(range(0, m_chunk_shape[di])); - di++; + return e.derived_cast().shape(); } - size_t ci = 0; - for (auto& chunk: m_chunks) + }; + + template class OP> + struct chunk_helper_impl>> + { + using is_chunked = std::true_type; + static const auto& chunk_shape(const xexpression& e) { - noalias(chunk) = strided_view(e.derived_cast(), sv); - bool last_chunk = ci == m_chunks.size() - 1; - if (!last_chunk) + return e.derived_cast().chunk_shape(); + } + }; + + template + using chunk_helper = chunk_helper_impl; + } + + template + constexpr bool is_chunked(const xexpression&) + { + using return_type = typename detail::chunk_helper::is_chunked; + return return_type::value; + } + + /********************************* + * xchunked_array implementation * + *********************************/ + + template + template + inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape) + { + resize(std::forward(shape), std::forward(chunk_shape)); + } + + template + template + inline xchunked_array::xchunked_array(const xexpression& e) + : xchunked_array(e, detail::chunk_helper::chunk_shape(e)) + { + } + + template + template + inline xchunked_array::xchunked_array(const xexpression& e, S&& chunk_shape) + { + resize(e.derived_cast().shape(), std::forward(chunk_shape)); + xstrided_slice_vector sv(m_chunk_shape.size()); // element slice corresponding to chunk + std::transform(m_chunk_shape.begin(), m_chunk_shape.end(), sv.begin(), + [](auto size) { return range(0, size); }); + + shape_type ic(this->dimension()); // index of chunk, initialized to 0... + size_type ci = 0; + for (auto& chunk: m_chunks) + { + noalias(chunk) = strided_view(e.derived_cast(), sv); + bool last_chunk = ci == m_chunks.size() - 1; + if (!last_chunk) + { + size_type di = this->dimension() - 1; + while (true) { - di = dimension() - 1; - while (true) + if (ic[di] + 1 == m_chunks.shape()[di]) { - if (ic[di] + 1 == m_chunks.shape()[di]) + ic[di] = 0; + sv[di] = range(0, m_chunk_shape[di]); + if (di == 0) { - ic[di] = 0; - sv[di] = range(0, m_chunk_shape[di]); - if (di == 0) - break; - else - di--; - + break; } else { - ic[di] += 1; - sv[di] = range(ic[di] * m_chunk_shape[di], (ic[di] + 1) * m_chunk_shape[di]); - break; + di--; } + + } + else + { + ic[di] += 1; + sv[di] = range(ic[di] * m_chunk_shape[di], (ic[di] + 1) * m_chunk_shape[di]); + break; } } - ci += 1; } + ++ci; } + } - chunk_storage& chunks() - { - return m_chunks; - } - - template - xchunked_array(const xexpression& e) - { - const auto& chunk_shape = detail::chunk_helper::chunk_shape(e); - *this = xchunked_array(e, chunk_shape); - } + template + template + inline auto xchunked_array::operator=(const xexpression& e) -> self_type& + { + return semantic_base::operator=(e); + } - template - self_type& operator=(const xexpression& e) - { - return semantic_base::operator=(e); - } + template + inline auto xchunked_array::shape() const noexcept -> const shape_type& + { + return m_shape; + } - reference operator[](const xindex& index) - { - reference el = element(index.cbegin(), index.cend()); - return el; - } + template + inline auto xchunked_array::layout() const noexcept -> layout_type + { + return static_layout; + } - const_reference operator[](const xindex& index) const - { - const_reference const_el = element(index.cbegin(), index.cend()); - return const_el; - } + template + inline bool xchunked_array::is_contiguous() const noexcept + { + return false; + } + + template + template + inline auto xchunked_array::operator()(Idxs... idxs) -> reference + { + auto ii = get_indexes(idxs...); + auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); + return chunk.element(ii.second.cbegin(), ii.second.cend()); + } - template - inline reference element(It first, It last) - { - auto ii = get_indexes_dynamic(first, last); - auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); - return chunk.element(ii.second.begin(), ii.second.end()); - } + template + template + inline auto xchunked_array::operator()(Idxs... idxs) const -> const_reference + { + auto ii = get_indexes(idxs...); + auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); + return chunk.element(ii.second.cbegin(), ii.second.cend()); + } - template - inline const_reference element(It first, It last) const - { - auto ii = get_indexes_dynamic(first, last); - auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); - return chunk.element(ii.second.begin(), ii.second.end()); - } + template + template + inline auto xchunked_array::element(It first, It last) -> reference + { + auto ii = get_indexes_dynamic(first, last); + auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); + return chunk.element(ii.second.begin(), ii.second.end()); + } - size_type dimension() const - { - return shape().size(); - } + template + template + inline auto xchunked_array::element(It first, It last) const -> const_reference + { + auto ii = get_indexes_dynamic(first, last); + auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); + return chunk.element(ii.second.begin(), ii.second.end()); + } - shape_type chunk_shape() const - { - return m_chunk_shape; - } + template + template + inline bool xchunked_array::broadcast_shape(S& s, bool) const + { + return xt::broadcast_shape(shape(), s); + } - template - bool broadcast_shape(S& s, bool reuse_cache = false) const - { - return xt::broadcast_shape(shape(), s); - } + template + template + inline auto xchunked_array::stepper_begin(const S& shape) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset); + } - private: + template + template + inline auto xchunked_array::stepper_end(const S& shape, layout_type) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset, true); + } - chunk_storage m_chunks; - shape_type m_shape; - shape_type m_chunk_shape; + template + template + inline auto xchunked_array::stepper_begin(const S& shape) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset); + } - template - void resize(S& shape, S& chunk_shape) - { - // compute chunk number in each dimension (shape_of_chunks) - std::vector shape_of_chunks(shape.size()); - size_t di = 0; - for (auto s: shape) - { - size_t cn = s / chunk_shape[di]; - if (s % chunk_shape[di] > 0) - cn += 1; // edge chunk - shape_of_chunks[di] = cn; - di++; - } - // resize the xarray of chunks - m_chunks.resize(shape_of_chunks); - // resize each chunk - for (auto& c: m_chunks) - c.resize(chunk_shape); - m_shape = shape; - m_chunk_shape = chunk_shape; - } + template + template + inline auto xchunked_array::stepper_end(const S& shape, layout_type) const noexcept -> const_stepper + { + size_type offset = shape.size() - this->dimension(); + return const_stepper(this, offset, true); + } + + template + inline auto xchunked_array::chunks() -> chunk_storage_type& + { + return m_chunks; + } - template - inline std::pair, std::array> get_indexes(Idxs... idxs) const - { - auto chunk_indexes_packed = get_chunk_indexes(std::make_index_sequence(), idxs...); - auto chunk_indexes = unpack(chunk_indexes_packed); - auto indexes_of_chunk = chunk_indexes.first; - auto indexes_in_chunk = chunk_indexes.second; - return std::make_pair(indexes_of_chunk, indexes_in_chunk); - } + template + inline auto xchunked_array::chunks() const -> const chunk_storage_type& + { + return m_chunks; + } - template - std::pair get_chunk_indexes_in_dimension(size_t dim, Idx idx) const - { - size_t index_of_chunk = idx / m_chunk_shape[dim]; - size_t index_in_chunk = idx - index_of_chunk * m_chunk_shape[dim]; - return std::make_pair(index_of_chunk, index_in_chunk); - } + template + inline auto xchunked_array::chunk_shape() const -> const shape_type& + { + return m_chunk_shape; + } - template - std::array, sizeof...(Idxs)> - get_chunk_indexes(std::index_sequence, Idxs... idxs) const - { - std::array, sizeof...(Idxs)> chunk_indexes = {{get_chunk_indexes_in_dimension(dims, idxs)...}}; - return chunk_indexes; - } - template - std::pair, std::array> unpack(std::array &arr) const - { - std::array arr0; - std::array arr1; - for (size_t i = 0; i < N; ++i) + + template + template + inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) + { + // compute chunk number in each dimension (shape_of_chunks) + std::vector shape_of_chunks(shape.size()); + std::transform + ( + shape.cbegin(), shape.cend(), + chunk_shape.cbegin(), + shape_of_chunks.begin(), + [](auto s, auto cs) { - arr0[i] = std::get<0>(arr[i]); - arr1[i] = std::get<1>(arr[i]); + size_t cn = s / cs; + if (s % cs > 0) + cn += size_t(1); // edge_chunk + return cn; } - return std::make_pair(arr0, arr1); - } + ); - template - inline std::pair, std::vector> get_indexes_dynamic(It first, It last) const + // resize the xarray of chunks + m_chunks.resize(shape_of_chunks); + // resize each chunk + for (auto& c: m_chunks) { - std::vector indexes_of_chunk; - std::vector indexes_in_chunk; - std::pair chunk_index; - size_t dim = 0; - for (auto it = first; it != last; ++it) - { - chunk_index = get_chunk_indexes_in_dimension(dim, *it); - indexes_of_chunk.push_back(chunk_index.first); - indexes_in_chunk.push_back(chunk_index.second); - dim++; - } - return std::make_pair(indexes_of_chunk, indexes_in_chunk); + c.resize(chunk_shape); } - template - bool is_trivial_broadcast(const S& str) const noexcept - { - return false; - } - }; + m_shape = xtl::forward_sequence(shape); + m_chunk_shape = xtl::forward_sequence(chunk_shape); + } - template - template - inline auto xchunked_array::stepper_begin(const O& shape) const noexcept -> const_stepper + template + template + inline auto xchunked_array::get_indexes(Idxs... idxs) const -> indexes_type { - size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset); + auto chunk_indexes_packed = get_chunk_indexes(std::make_index_sequence(), idxs...); + return unpack(chunk_indexes_packed); } - template - template - inline auto xchunked_array::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper + template + template + inline std::pair xchunked_array::get_chunk_indexes_in_dimension(size_t dim, Idx idx) const { - size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset, true); + size_t index_of_chunk = idx / m_chunk_shape[dim]; + size_t index_in_chunk = idx - index_of_chunk * m_chunk_shape[dim]; + return std::make_pair(index_of_chunk, index_in_chunk); } - template - template - inline auto xchunked_array::stepper_begin(const O& shape) noexcept -> stepper + template + template + inline auto xchunked_array::get_chunk_indexes(std::index_sequence, Idxs... idxs) const + -> chunk_indexes_type { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset); + chunk_indexes_type chunk_indexes = {{get_chunk_indexes_in_dimension(dims, idxs)...}}; + return chunk_indexes; } - template - template - inline auto xchunked_array::stepper_end(const O& shape, layout_type) noexcept -> stepper + template + template + inline auto xchunked_array::unpack(const std::array &arr) const -> static_indexes_type { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset, true); + std::array arr0; + std::array arr1; + for (size_t i = 0; i < N; ++i) + { + arr0[i] = std::get<0>(arr[i]); + arr1[i] = std::get<1>(arr[i]); + } + return std::make_pair(arr0, arr1); + } + + template + template + inline auto xchunked_array::get_indexes_dynamic(It first, It last) const -> dynamic_indexes_type + { + auto size = static_cast(std::distance(first, last)); + std::vector indexes_of_chunk(size); + std::vector indexes_in_chunk(size); + for (auto dim = 0; dim < size; ++dim) + { + auto chunk_index = get_chunk_indexes_in_dimension(dim, *first++); + indexes_of_chunk[dim] = chunk_index.first; + indexes_in_chunk[dim] = chunk_index.second; + } + return std::make_pair(indexes_of_chunk, indexes_in_chunk); } } From 5f3c37de792bf39917242ff1600267b540fc3c7e Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Sat, 15 Aug 2020 00:01:19 +0200 Subject: [PATCH 068/606] xchunk_store_manager code cleanup --- include/xtensor/xchunk_store_manager.hpp | 357 +++++++++++++---------- include/xtensor/xchunked_array.hpp | 3 +- 2 files changed, 199 insertions(+), 161 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index d33bc90f8..6fd4cfd34 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -10,16 +10,21 @@ namespace xt { + + /************************************ + * xchunk_store_manager declaration * + ************************************/ + template class xchunk_store_manager; template struct xcontainer_inner_types> { - using const_reference = const EC&; + using storage_type = EC; using reference = EC&; + using const_reference = const EC&; using size_type = std::size_t; - using storage_type = EC; using temporary_type = xchunk_store_manager; }; @@ -27,8 +32,8 @@ namespace xt struct xiterable_inner_types> { using inner_shape_type = std::vector; - using const_stepper = xindexed_stepper, true>; using stepper = xindexed_stepper, false>; + using const_stepper = xindexed_stepper, true>; }; template @@ -37,184 +42,143 @@ namespace xt { public: - using const_reference = const EC&; - using reference = EC&; using self_type = xchunk_store_manager; - using iterable_base = xconst_iterable; - using const_stepper = typename iterable_base::const_stepper; - using stepper = typename iterable_base::stepper; using inner_types = xcontainer_inner_types; - using size_type = typename inner_types::size_type; using storage_type = typename inner_types::storage_type; using value_type = storage_type; + using reference = EC&; + using const_reference = const EC&; using pointer = value_type*; using const_pointer = const value_type*; + using size_type = typename inner_types::size_type; using difference_type = std::ptrdiff_t; - using shape_type = std::vector; + using iterable_base = xconst_iterable; + using stepper = typename iterable_base::stepper; + using const_stepper = typename iterable_base::const_stepper; + using shape_type = typename iterable_base::inner_shape_type; - template - const_stepper stepper_begin(const O& shape) const noexcept; - template - const_stepper stepper_end(const O& shape, layout_type) const noexcept; + xchunk_store_manager(); + ~xchunk_store_manager() = default; + + xchunk_store_manager(const xchunk_store_manager&) = default; + xchunk_store_manager& operator=(const xchunk_store_manager&) = default; + + xchunk_store_manager(xchunk_store_manager&&) = default; + xchunk_store_manager& operator=(xchunk_store_manager&&) = default; + + const shape_type& shape() const noexcept; + + template + reference operator()(Idxs... idxs); + + template + const_reference operator()(Idxs... idxs) const; + + template + reference element(It first, It last); + + template + const_reference element(It first, It last) const; template stepper stepper_begin(const O& shape) noexcept; template stepper stepper_end(const O& shape, layout_type) noexcept; + + template + const_stepper stepper_begin(const O& shape) const noexcept; + template + const_stepper stepper_end(const O& shape, layout_type) const noexcept; - const shape_type& shape() const - { - return m_shape; - } - - xchunk_store_manager() - { - // default pool size is 1 - // so that first chunk is always resized to the chunk shape - m_chunk_pool.resize(1); - m_index_pool.resize(1); - m_unload_index = 0; - } - - void set_pool_size(std::size_t n) - { - // first chunk always has the correct shape - // get the shape before resizing the pool - auto chunk_shape = m_chunk_pool[0].array().shape(); - m_chunk_pool.resize(n); - m_index_pool.resize(n); - m_unload_index = 0; - // resize the pool chunks - for (auto& chunk: m_chunk_pool) - { - chunk.resize(chunk_shape); - } - } + template + void resize(S&& shape); - void flush() - { - for (auto& chunk: m_chunk_pool) - { - chunk.flush(); - } - } + void set_pool_size(std::size_t n); + void flush(); template - EC& map_file_array(I first, I last) - { - std::string path; - std::vector index; - for (auto it = first; it != last; ++it) - { - if (!path.empty()) - { - path.append("."); - } - path.append(std::to_string(*it)); - index.push_back(*it); - } - if (index.empty()) - { - return m_chunk_pool[0]; - } - else - { - // check if the chunk is already loaded in memory - const auto it1 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), index); - std::size_t i; - if (it1 != m_index_pool.cend()) - { - i = std::distance(m_index_pool.cbegin(), it1); - return m_chunk_pool[i]; - } - // if not, find a free chunk in the pool - std::vector empty_index; - const auto it2 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), empty_index); - if (it2 != m_index_pool.cend()) - { - i = std::distance(m_index_pool.cbegin(), it2); - m_chunk_pool[i].set_path(path); - m_index_pool[i] = index; - return m_chunk_pool[i]; - } - // no free chunk, take one (which will thus be unloaded) - // fairness is guaranteed through the use of a walking index - m_chunk_pool[m_unload_index].set_path(path); - m_index_pool[m_unload_index] = index; - auto& chunk = m_chunk_pool[m_unload_index]; - m_unload_index = (m_unload_index + 1) % m_index_pool.size(); - return chunk; - } - } + reference map_file_array(I first, I last); - template - inline const_reference operator()(Idxs... idxs) const - { - auto index = get_indexes(idxs...); - return map_file_array(index.cbegin(), index.cend()); - } + private: template - inline reference operator()(Idxs... idxs) - { - auto index = get_indexes(idxs...); - return map_file_array(index.cbegin(), index.cend()); - } + std::array get_indexes(Idxs... idxs) const; - xchunk_store_manager(const xchunk_store_manager&) = default; - xchunk_store_manager& operator=(const xchunk_store_manager&) = default; + using chunk_pool_type = std::vector; + using index_pool_type = std::vector; - xchunk_store_manager(xchunk_store_manager&&) = default; - xchunk_store_manager& operator=(xchunk_store_manager&&) = default; - - reference operator[](const xindex& index) - { - return map_file_array(index.cbegin(), index.cend()); - } + shape_type m_shape; + chunk_pool_type m_chunk_pool; + index_pool_type m_index_pool; + std::size_t m_unload_index; + }; - const_reference operator[](const xindex& index) const - { - return map_file_array(index.cbegin(), index.cend()); - } + /*************************************** + * xchunk_store_manager implementation * + ***************************************/ - template - inline reference element(It first, It last) - { - return map_file_array(first, last); - } + template + inline xchunk_store_manager::xchunk_store_manager() + : m_shape() + // default pool size is 1 + // so that first chunk is always resized to the chunk shape + , m_chunk_pool(1u) + , m_index_pool(1u) + , m_unload_index(0u) + { + } - template - inline const_reference element(It first, It last) const - { - return map_file_array(first, last); - } + template + inline auto xchunk_store_manager::shape() const noexcept -> const shape_type& + { + return m_shape; + } - size_type dimension() const - { - return shape().size(); - } + template + template + inline auto xchunk_store_manager::operator()(Idxs... idxs) -> reference + { + auto index = get_indexes(idxs...); + return map_file_array(index.cbegin(), index.cend()); + } - template - void resize(S& shape) - { - // don't resize according to total number of chunks - // instead the pool manages a number of in-memory chunks - } + template + template + inline auto xchunk_store_manager::operator()(Idxs... idxs) const -> const_reference + { + auto index = get_indexes(idxs...); + return map_file_array(index.cbegin(), index.cend()); + } - private: + template + template + inline auto xchunk_store_manager::element(It first, It last) -> reference + { + return map_file_array(first, last); + } - shape_type m_shape; - std::vector m_chunk_pool; - std::vector> m_index_pool; - std::size_t m_unload_index; + template + template + inline auto xchunk_store_manager::element(It first, It last) const -> const_reference + { + return map_file_array(first, last); + } + + template + template + inline auto xchunk_store_manager::stepper_begin(const O& shape) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset); + } - template - inline std::array get_indexes(Idxs... idxs) const - { - std::array indexes = {{idxs...}}; - return indexes; - } - }; + template + template + inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) noexcept -> stepper + { + size_type offset = shape.size() - this->dimension(); + return stepper(this, offset, true); + } template template @@ -233,19 +197,94 @@ namespace xt } template - template - inline auto xchunk_store_manager::stepper_begin(const O& shape) noexcept -> stepper + template + inline void xchunk_store_manager::resize(S&&) { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset); + // don't resize according to total number of chunks + // instead the pool manages a number of in-memory chunks + } + + template + inline void xchunk_store_manager::set_pool_size(std::size_t n) + { + // first chunk always has the correct shape + // get the shape before resizing the pool + auto chunk_shape = m_chunk_pool[0].array().shape(); + m_chunk_pool.resize(n); + m_index_pool.resize(n); + m_unload_index = 0; + // resize the pool chunks + for (auto& chunk: m_chunk_pool) + { + chunk.resize(chunk_shape); + } } template - template - inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) noexcept -> stepper + inline void xchunk_store_manager::flush() { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset, true); + for (auto& chunk: m_chunk_pool) + { + chunk.flush(); + } + } + + template + template + inline auto xchunk_store_manager::map_file_array(I first, I last) -> reference + { + std::string path; + std::vector index; + for (auto it = first; it != last; ++it) + { + if (!path.empty()) + { + path.append("."); + } + path.append(std::to_string(*it)); + index.push_back(*it); + } + if (index.empty()) + { + return m_chunk_pool[0]; + } + else + { + // check if the chunk is already loaded in memory + const auto it1 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), index); + std::size_t i; + if (it1 != m_index_pool.cend()) + { + i = std::distance(m_index_pool.cbegin(), it1); + return m_chunk_pool[i]; + } + // if not, find a free chunk in the pool + std::vector empty_index; + const auto it2 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), empty_index); + if (it2 != m_index_pool.cend()) + { + i = std::distance(m_index_pool.cbegin(), it2); + m_chunk_pool[i].set_path(path); + m_index_pool[i] = index; + return m_chunk_pool[i]; + } + // no free chunk, take one (which will thus be unloaded) + // fairness is guaranteed through the use of a walking index + m_chunk_pool[m_unload_index].set_path(path); + m_index_pool[m_unload_index] = index; + auto& chunk = m_chunk_pool[m_unload_index]; + m_unload_index = (m_unload_index + 1) % m_index_pool.size(); + return chunk; + } + } + + template + template + inline std::array + xchunk_store_manager::get_indexes(Idxs... idxs) const + { + std::array indexes = {{idxs...}}; + return indexes; } } diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 971a1c50e..ca126efdf 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -68,6 +68,7 @@ namespace xt template xchunked_array(S&& shape, S&& chunk_shape); + ~xchunked_array() = default; xchunked_array(const xchunked_array&) = default; xchunked_array& operator=(const xchunked_array&) = default; @@ -379,8 +380,6 @@ namespace xt return m_chunk_shape; } - - template template inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) From c21a28e60c0476ca57ffe7dbd018cd27b2d57bb1 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Sat, 15 Aug 2020 00:10:05 +0200 Subject: [PATCH 069/606] Removed refernces to outdated is_trivial_broadcast method in the documentation --- docs/source/developer/concepts.rst | 3 --- docs/source/developer/implementation_classes.rst | 3 --- docs/source/external-structures.rst | 6 ------ 3 files changed, 12 deletions(-) diff --git a/docs/source/developer/concepts.rst b/docs/source/developer/concepts.rst index 298be2ec2..371470351 100644 --- a/docs/source/developer/concepts.rst +++ b/docs/source/developer/concepts.rst @@ -263,9 +263,6 @@ methods: template bool broadcast_shape(const S& shape) const; - template - bool is_trivial_broadcast(const S& strides) const; - Lower-level methods are also provided, meant for optimized assignment and BLAS bindings. They are covered in the :ref:`xtensor-assign-label` section. diff --git a/docs/source/developer/implementation_classes.rst b/docs/source/developer/implementation_classes.rst index 8139a4c9b..803d13390 100644 --- a/docs/source/developer/implementation_classes.rst +++ b/docs/source/developer/implementation_classes.rst @@ -77,9 +77,6 @@ totally: template bool broadcast_shape(const S& shape) const; - template - bool is_trivial_broadcast(const S& strides) const; - **data access methods** .. code:: diff --git a/docs/source/external-structures.rst b/docs/source/external-structures.rst index 4df8f34bd..f84294fa5 100644 --- a/docs/source/external-structures.rst +++ b/docs/source/external-structures.rst @@ -440,12 +440,6 @@ This part is relatively straightforward: return xt::broadcast_shape(shape(), s); } - template - bool is_trivial_broadcast(const S& str) const noexcept - { - return false; - } - Implement resize overloads ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From a3e4cf0334fe3d379041acf7cfe031ea6c8f6159 Mon Sep 17 00:00:00 2001 From: nb Date: Tue, 18 Aug 2020 18:18:10 -0400 Subject: [PATCH 070/606] fixed sturct typo in bindings.rst doc --- docs/source/bindings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/bindings.rst b/docs/source/bindings.rst index d215e360c..52180d89d 100644 --- a/docs/source/bindings.rst +++ b/docs/source/bindings.rst @@ -246,7 +246,7 @@ for C: // Equivalent to is_tensor::value || is_array::value template - sturct is_container : xtl::disjunction, is_array> + struct is_container : xtl::disjunction, is_array> { }; From 80fa87ea10b2ca1ee73c1eac9753577090b471e0 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 20 Aug 2020 01:59:52 +0200 Subject: [PATCH 071/606] Refactored xfile_array.hpp --- include/xtensor/xchunk_store_manager.hpp | 2 +- include/xtensor/xchunked_array.hpp | 2 +- include/xtensor/xfile_array.hpp | 646 +++++++++++++++-------- 3 files changed, 415 insertions(+), 235 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 6fd4cfd34..e4853828e 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -209,7 +209,7 @@ namespace xt { // first chunk always has the correct shape // get the shape before resizing the pool - auto chunk_shape = m_chunk_pool[0].array().shape(); + auto chunk_shape = m_chunk_pool[0].storage().shape(); m_chunk_pool.resize(n); m_index_pool.resize(n); m_unload_index = 0; diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index ca126efdf..7deb6a342 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -459,7 +459,7 @@ namespace xt auto size = static_cast(std::distance(first, last)); std::vector indexes_of_chunk(size); std::vector indexes_in_chunk(size); - for (auto dim = 0; dim < size; ++dim) + for (size_t dim = 0; dim < size; ++dim) { auto chunk_index = get_chunk_indexes_in_dimension(dim, *first++); indexes_of_chunk[dim] = chunk_index.first; diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index 83c2c3fbe..8f59b2703 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -10,338 +10,518 @@ namespace xt { - namespace detail + + template + class xfile_reference { - // Workaround for VS2015 - template - using try_path = decltype(std::declval().path()); + public: - template class OP, class = void> - struct file_helper_impl - { - static const char* path(const xexpression& e) - { - return ""; - } - using is_stored = std::false_type; - }; + using self_type = xfile_reference; + using const_reference = const T&; - template class OP> - struct file_helper_impl>> - { - static const char* path(const xexpression& e) - { - return e.derived_cast().path(); - } - using is_stored = std::true_type; - }; + xfile_reference(T& value, bool& dirty); + ~xfile_reference() = default; - template - using file_helper = file_helper_impl; - } + xfile_reference(const xfile_reference&) = default; + xfile_reference(xfile_reference&&) = default; - template - constexpr bool is_stored(const xexpression& e) - { - using return_type = typename detail::file_helper::is_stored; - return return_type::value; - } + self_type& operator=(const self_type&); + self_type& operator=(self_type&&); - template - class xfile_reference - { - public: + template + self_type& operator=(const V&); - xfile_reference(EC& value, bool& array_dirty) - { - m_pvalue = &value; - m_parray_dirty = &array_dirty; - } + template + self_type& operator+=(const V&); - operator const EC() const - { - return *m_pvalue; - } + template + self_type& operator-=(const V&); - operator EC() - { - return *m_pvalue; - } + template + self_type& operator*=(const V&); - EC operator=(const EC value) - { - if (value != *m_pvalue) - { - *m_parray_dirty = true; - *m_pvalue = value; - } - return *m_pvalue; - } + template + self_type& operator/=(const V&); - private: + operator const_reference() const; - EC* m_pvalue; - bool* m_parray_dirty; + private: + T& m_value; + bool& m_dirty; }; - template - class xfile_array; + template + class xfile_array_container; - template - struct xcontainer_inner_types> + template + struct xcontainer_inner_types> { - using const_reference = const EC&; - using reference = xfile_reference; - using size_type = std::size_t; - using storage_type = EC; - using temporary_type = xfile_array; + using storage_type = E; + using value_type = typename storage_type::value_type; + using reference = xfile_reference; + using const_reference = typename storage_type::const_reference; + using size_type = typename storage_type::size_type; + using temporary_type = xfile_array_container; }; - template - struct xiterable_inner_types> + template + struct xiterable_inner_types> { - using inner_shape_type = typename xarray::shape_type; - using const_stepper = xindexed_stepper, true>; - using stepper = xindexed_stepper, false>; + using inner_shape_type = typename E::shape_type; + using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; }; - template - class xfile_array: public xaccessible>, - public xiterable>, - public xcontainer_semantic> + template + class xfile_array_container : public xaccessible>, + public xiterable>, + public xcontainer_semantic> { public: - using const_reference = const EC&; - using reference = xfile_reference; - using self_type = xfile_array; + using self_type = xfile_array_container; using semantic_base = xcontainer_semantic; using iterable_base = xconst_iterable; - using const_stepper = typename iterable_base::const_stepper; - using stepper = typename iterable_base::stepper; using inner_types = xcontainer_inner_types; - using size_type = typename inner_types::size_type; using storage_type = typename inner_types::storage_type; - using value_type = storage_type; - using pointer = value_type*; - using const_pointer = const value_type*; - using difference_type = std::ptrdiff_t; - using shape_type = typename xarray::shape_type; + using value_type = typename storage_type::value_type; + using reference = typename inner_types::reference; + using const_reference = typename inner_types::const_reference; + using pointer = typename storage_type::pointer; + using const_pointer = typename storage_type::const_pointer; + using size_type = typename inner_types::size_type; + using difference_type = typename storage_type::difference_type; + using shape_type = typename storage_type::shape_type; + using strides_type = typename storage_type::strides_type; + using stepper = typename iterable_base::stepper; + using const_stepper = typename iterable_base::const_stepper; using temporary_type = typename inner_types::temporary_type; using bool_load_type = xt::bool_load_type; static constexpr layout_type static_layout = layout_type::dynamic; - template - const_stepper stepper_begin(const O& shape) const noexcept; - template - const_stepper stepper_end(const O& shape, layout_type) const noexcept; + xfile_array_container() = default; + ~xfile_array_container(); + + xfile_array_container(const self_type&) = default; + self_type& operator=(const self_type&) = default; + + xfile_array_container(self_type&&) = default; + self_type& operator=(self_type&&) = default; + + template + xfile_array_container(const xexpression& e); + + template + xfile_array_container(const xexpression& e, const std::string& path); + + template + self_type& operator=(const xexpression& e); + + size_type size() const noexcept; + const shape_type& shape() const noexcept; + layout_type layout() const noexcept; + bool is_contiguous() const noexcept; + + template + void resize(S&& shape, bool force = false); + template + void resize(S&& shape, layout_type l); + template + void resize(S&& shape, const strides_type& strides); + + template + self_type& reshape(S&& shape, layout_type layout = static_layout) &; + + template + self_type& reshape(std::initializer_list shape, layout_type layout = static_layout) &; + + template + reference operator()(Idxs... idxs); + + template + const_reference operator()(Idxs... idxs) const; + + template + reference element(It first, It last); + + template + const_reference element(It first, It last) const; + + storage_type& storage() noexcept; + const storage_type& storage() const noexcept; + + template + bool broadcast_shape(S& s, bool reuse_cache = false) const; + + template + bool has_linear_assign(const S& strides) const noexcept; template stepper stepper_begin(const O& shape) noexcept; template stepper stepper_end(const O& shape, layout_type) noexcept; - const auto& shape() const - { - return m_array.shape(); - } + template + const_stepper stepper_begin(const O& shape) const noexcept; + template + const_stepper stepper_end(const O& shape, layout_type) const noexcept; - inline layout_type layout() const noexcept - { - return static_layout; - } + const std::string& path() const noexcept; + void set_path(std::string& path); - inline bool is_contiguous() const noexcept - { - return false; - } + void flush(); - xfile_array() {} + private: - ~xfile_array() - { - flush(); - } + E m_storage; + bool m_dirty; + IOH m_io_handler; + std::string m_path; + }; - void flush() - { - if (m_array_dirty) - { - m_io_handler.write(m_array, m_path); - m_array_dirty = false; - } - } + template ::size_type>> + using xfile_array = xfile_array_container, IOH>; - xarray& array() - { - return m_array; - } + /********************************** + * xfile_reference implementation * + **********************************/ - void set_path(const char* path) + template + inline xfile_reference::xfile_reference(T& value, bool& dirty) + : m_value(value), m_dirty(dirty) + { + } + + template + template + inline auto xfile_reference::operator=(const V& v) -> self_type& + { + if (v != m_value) { - std::string p(path); - set_path(p); + m_value = v; + m_dirty = true; } + return *this; + } - void set_path(std::string& path) + template + template + inline auto xfile_reference::operator+=(const V& v) -> self_type& + { + if (v != T(0)) { - if (path != m_path) - { - // maybe write to old file - if (m_array_dirty) - { - m_io_handler.write(m_array, m_path); - m_array_dirty = false; - } - m_path = path; - // read new file - m_io_handler.read(m_array, path); - } + m_value += v; + m_dirty = true; } + return *this; + } - template - void resize(S& shape) + template + template + inline auto xfile_reference::operator-=(const V& v) -> self_type& + { + if (v != T(0)) { - m_array.resize(shape); - m_array = broadcast(0, shape); + m_value -= v; + m_dirty = true; } + return *this; + } - template - inline reference operator()(Idxs... idxs) + template + template + inline auto xfile_reference::operator*=(const V& v) -> self_type& + { + if (v != T(1)) { - auto index = get_indexes(idxs...); - return reference(m_array.element(index.cbegin(), index.cend()), m_array_dirty); + m_value *= v; + m_dirty = true; } + return *this; + } - template - inline const_reference operator()(Idxs... idxs) const + template + template + inline auto xfile_reference::operator/=(const V& v) -> self_type& + { + if (v != T(1)) { - auto index = get_indexes(idxs...); - return m_array.element(index.cbegin(), index.cend()); + m_value /= v; + m_dirty = true; } + return *this; + } - xfile_array(const xfile_array&) = default; - xfile_array& operator=(const xfile_array&) = default; + template + inline xfile_reference::operator const_reference() const + { + return m_value; + } - xfile_array(xfile_array&&) = default; - xfile_array& operator=(xfile_array&&) = default; + /**************************************** + * xfile_array_container implementation * + ****************************************/ + namespace detail + { + // Workaround for VS2015 template - xfile_array(const xexpression& e, const char* path) + using try_path = decltype(std::declval().path()); + + template class OP, class = void> + struct file_helper_impl { - set_path(path); - m_array_dirty = true; - const auto& shape = e.derived_cast().shape(); - m_array.resize(shape); - xstrided_slice_vector sv; - for (auto i = 0; i < dimension(); i++) - sv.push_back(all()); - noalias(m_array) = strided_view(e.derived_cast(), sv); - } + using is_stored = std::false_type; - template - xfile_array(const xexpression& e) + static const char* path(const xexpression& e) + { + return ""; + } + }; + + template class OP> + struct file_helper_impl>> { - const char* path = detail::file_helper::path(e); - *this = xfile_array(e, path); - } + using is_stored = std::true_type; + + static const char* path(const xexpression& e) + { + return e.derived_cast().path(); + } + }; template - self_type& operator=(const xexpression& e) - { - return semantic_base::operator=(e); - } + using file_helper = file_helper_impl; + } - reference operator[](const xindex& index) - { - return reference(m_array.element(index.cbegin(), index.cend()), m_array_dirty); - } + template + constexpr bool is_stored(const xexpression& e) + { + using return_type = typename detail::file_helper::is_stored; + return return_type::value; + } - const_reference operator[](const xindex& index) const - { - return m_array.element(index.cbegin(), index.cend()); - } + template + inline xfile_array_container::~xfile_array_container() + { + flush(); + } - template - inline reference element(It first, It last) - { - return reference(m_array.element(first, last), m_array_dirty); - } + template + template + inline xfile_array_container::xfile_array_container(const xexpression& e) + : m_storage(e) + , m_dirty(true) + , m_io_handler() + , m_path(detail::file_helper::path(e)) + { + } - template - inline const_reference element(It first, It last) const - { - return m_array.element(first, last); - } + template + template + inline xfile_array_container::xfile_array_container(const xexpression& e, const std::string& path) + : m_storage(e) + , m_dirty(true) + , m_io_handler() + , m_path(path) + { + } + + template + template + inline auto xfile_array_container::operator=(const xexpression& e) -> self_type& + { + return semantic_base::operator=(e); + } - size_type dimension() const - { - return shape().size(); - } + template + inline auto xfile_array_container::size() const noexcept -> size_type + { + return m_storage.size(); + } + + template + inline auto xfile_array_container::shape() const noexcept -> const shape_type& + { + return m_storage.shape(); + } + + template + inline auto xfile_array_container::layout() const noexcept -> layout_type + { + return m_storage.layout(); + } - const char* path() const - { - return m_path.c_str(); - } + template + inline bool xfile_array_container::is_contiguous() const noexcept + { + return m_storage.is_contiguous(); + } - template - bool broadcast_shape(S& s, bool reuse_cache = false) const - { - return xt::broadcast_shape(shape(), s); - } + template + template + inline void xfile_array_container::resize(S&& shape, bool force) + { + m_storage.resize(std::forward(shape), force); + m_dirty = true; + } - private: + template + template + inline void xfile_array_container::resize(S&& shape, layout_type l) + { + m_storage.resize(std::forward(shape), l); + m_dirty = true; + } - xarray m_array; - bool m_array_dirty; - io_handler m_io_handler; - std::string m_path; + template + template + inline void xfile_array_container::resize(S&& shape, const strides_type& strides) + { + m_storage.resize(std::forward(shape), strides); + m_dirty = true; + } + + template + template + inline auto xfile_array_container::reshape(S&& shape, layout_type layout) & -> self_type& + { + m_storage.reshape(std::forward(shape), layout); + m_dirty = true; + return *this; + } - template - inline std::array get_indexes(Idxs... idxs) const - { - std::array indexes = {{idxs...}}; - return indexes; - } + template + template + inline auto xfile_array_container::reshape(std::initializer_list shape, layout_type layout) & -> self_type& + { + m_storage.reshape(shape, layout); + m_dirty = true; + return *this; + } - template - bool is_trivial_broadcast(const S& str) const noexcept - { - return false; - } - }; + template + template + inline auto xfile_array_container::operator()(Idxs... idxs) -> reference + { + return reference(m_storage(idxs...), m_dirty); + } + + template + template + inline auto xfile_array_container::operator()(Idxs... idxs) const -> const_reference + { + return m_storage(idxs...); + } + + template + template + inline auto xfile_array_container::element(It first, It last) -> reference + { + return reference(m_storage.element(first, last), m_dirty); + } + + template + template + inline auto xfile_array_container::element(It first, It last) const -> const_reference + { + return m_storage.element(first, last); + } - template + template + inline auto xfile_array_container::storage() noexcept -> storage_type& + { + return m_storage; + } + + template + inline auto xfile_array_container::storage() const noexcept -> const storage_type& + { + return m_storage; + } + + template + template + inline bool xfile_array_container::broadcast_shape(S& s, bool reuse_cache) const + { + return m_storage.broadcast_shape(s, reuse_cache); + } + + template + template + inline bool xfile_array_container::has_linear_assign(const S& strides) const noexcept + { + return m_storage.has_linear_assign(strides); + } + + template template - inline auto xfile_array::stepper_begin(const O& shape) const noexcept -> const_stepper + inline auto xfile_array_container::stepper_begin(const O& shape) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset); + return stepper(this, offset); } - template + template template - inline auto xfile_array::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper + inline auto xfile_array_container::stepper_end(const O& shape, layout_type) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset, true); + return stepper(this, offset, true); } - template + template template - inline auto xfile_array::stepper_begin(const O& shape) noexcept -> stepper + inline auto xfile_array_container::stepper_begin(const O& shape) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); - return stepper(this, offset); + return const_stepper(this, offset); } - template + template template - inline auto xfile_array::stepper_end(const O& shape, layout_type) noexcept -> stepper + inline auto xfile_array_container::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); - return stepper(this, offset, true); + return const_stepper(this, offset, true); + } + + template + inline const std::string& xfile_array_container::path() const noexcept + { + return m_path; + } + + template + inline void xfile_array_container::set_path(std::string& path) + { + if (path != m_path) + { + // maybe write to old file + if (m_dirty) + { + m_io_handler.write(m_storage, m_path); + m_dirty = false; + } + m_path = path; + // read new file + m_io_handler.read(m_storage, path); + } + } + + template + inline void xfile_array_container::flush() + { + if (m_dirty) + { + m_io_handler.write(m_storage, m_path); + m_dirty = false; + } } } From 6cc2c1bac8bec1a37d47ab499093d2d4ff93ee48 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 20 Aug 2020 10:26:45 +0200 Subject: [PATCH 072/606] Added simd accessors to xfile_array_container --- include/xtensor/xfile_array.hpp | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index 8f59b2703..2b06ffb17 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -167,6 +167,19 @@ namespace xt template const_stepper stepper_end(const O& shape, layout_type) const noexcept; + reference data_element(size_type i); + const_reference data_element(size_type i) const; + + template + using simd_return_type = xt_simd::simd_return_type; + + template + void store_simd(size_type i, const simd& e); + template ::size> + container_simd_return_type_t + load_simd(size_type i) const; + const std::string& path() const noexcept; void set_path(std::string& path); @@ -491,6 +504,39 @@ namespace xt return const_stepper(this, offset, true); } + template + inline auto xfile_array_container::data_element(size_type i) -> reference + { + return reference(m_storage.data_element(i), m_dirty); + } + + template + inline auto xfile_array_container::data_element(size_type i) const -> const_reference + { + return m_storage.element(i); + } + + template + template + inline void xfile_array_container::store_simd(size_type i, const simd& e) + { + m_storage.store_simd(i, e); + m_dirty = true; + } + + template + template + inline auto xfile_array_container::load_simd(size_type i) const + -> container_simd_return_type_t + { + return m_storage.load_simd(i); + } + + + + + + template inline const std::string& xfile_array_container::path() const noexcept { From 95a405a4d43768581d75b4e77a48b29354b9bd9f Mon Sep 17 00:00:00 2001 From: David Brochart Date: Wed, 19 Aug 2020 10:35:44 +0200 Subject: [PATCH 073/606] Abstract file format through a format class --- include/xtensor/xchunk_store_manager.hpp | 41 ++++++++++++++++++++---- include/xtensor/xcsv.hpp | 28 ++++++++++++++++ include/xtensor/xdisk_io_handler.hpp | 41 ++++++++++++++++-------- include/xtensor/xfile_array.hpp | 25 +++++++++------ test/test_xchunked_array.cpp | 9 +++--- 5 files changed, 109 insertions(+), 35 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index e4853828e..2b185eae5 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -10,7 +10,7 @@ namespace xt { - + /************************************ * xchunk_store_manager declaration * ************************************/ @@ -84,7 +84,7 @@ namespace xt stepper stepper_begin(const O& shape) noexcept; template stepper stepper_end(const O& shape, layout_type) noexcept; - + template const_stepper stepper_begin(const O& shape) const noexcept; template @@ -94,8 +94,12 @@ namespace xt void resize(S&& shape); void set_pool_size(std::size_t n); + void set_directory(const char* directory); void flush(); + template + void configure_format(C& config); + template reference map_file_array(I first, I last); @@ -111,6 +115,7 @@ namespace xt chunk_pool_type m_chunk_pool; index_pool_type m_index_pool; std::size_t m_unload_index; + std::string m_directory; }; /*************************************** @@ -163,7 +168,7 @@ namespace xt { return map_file_array(first, last); } - + template template inline auto xchunk_store_manager::stepper_begin(const O& shape) noexcept -> stepper @@ -203,7 +208,7 @@ namespace xt // don't resize according to total number of chunks // instead the pool manages a number of in-memory chunks } - + template inline void xchunk_store_manager::set_pool_size(std::size_t n) { @@ -229,21 +234,43 @@ namespace xt } } + template + template + void xchunk_store_manager::configure_format(C& config) + { + for (auto& chunk: m_chunk_pool) + { + chunk.configure_format(config); + } + } + + template + void xchunk_store_manager::set_directory(const char* directory) + { + m_directory = directory; + if (m_directory.back() != '/') + { + m_directory.append("/"); + } + } + template template inline auto xchunk_store_manager::map_file_array(I first, I last) -> reference { std::string path; + std::string fname; std::vector index; for (auto it = first; it != last; ++it) { - if (!path.empty()) + if (!fname.empty()) { - path.append("."); + fname.append("."); } - path.append(std::to_string(*it)); + fname.append(std::to_string(*it)); index.push_back(*it); } + path = m_directory + fname; if (index.empty()) { return m_chunk_pool[0]; diff --git a/include/xtensor/xcsv.hpp b/include/xtensor/xcsv.hpp index 4e49d0228..96de5a1f3 100644 --- a/include/xtensor/xcsv.hpp +++ b/include/xtensor/xcsv.hpp @@ -198,6 +198,34 @@ namespace xt } } } + + struct xcsv_config + { + char delimiter; + std::size_t skip_rows; + std::ptrdiff_t max_rows; + std::string comments; + + xcsv_config() + : delimiter(',') + , skip_rows(0) + , max_rows(-1) + , comments("#") + { + } + }; + + template + auto load_file(std::istream& stream, const xcsv_config& config) + { + return load_csv(stream, config.delimiter, config.skip_rows, config.max_rows, config.comments); + } + + template + void dump_file(std::ostream& stream, const xexpression &e, const xcsv_config&) + { + dump_csv(stream, e); + } } #endif diff --git a/include/xtensor/xdisk_io_handler.hpp b/include/xtensor/xdisk_io_handler.hpp index aff422ad4..9991166d0 100644 --- a/include/xtensor/xdisk_io_handler.hpp +++ b/include/xtensor/xdisk_io_handler.hpp @@ -2,40 +2,53 @@ #define XTENSOR_DISK_IO_HANDLER_HPP #include "xarray.hpp" -#include "xcsv.hpp" +#include "xexpression.hpp" namespace xt { - template + template class xdisk_io_handler { public: - - void write(xarray& array, std::string& path) + template + void write(xexpression& expression, std::string& path) { - std::ofstream m_out_file; - m_out_file.open(path); + std::ofstream m_out_file(path, std::ofstream::binary); if (m_out_file.is_open()) { - dump_csv(m_out_file, array); - m_out_file.close(); + dump_file(m_out_file, expression, m_format_config); + } + else + { + std::runtime_error("write: failed to open file " + path); } } - void read(xarray& array, std::string& path) + template + void read(ET& array, std::string& path) { - std::ifstream m_in_file; - m_in_file.open(path); + // not all formats store the shape (e.g. Blosc) + // so we reshape after loading + std::ifstream m_in_file(path, std::ifstream::binary); + const auto shape = array.shape(); if (m_in_file.is_open()) { - array = load_csv(m_in_file); - m_in_file.close(); + array = load_file(m_in_file, m_format_config); + array.reshape(shape); } else { - array = broadcast(0, array.shape()); + array = zeros(shape); } } + + void configure_format(C& format_config) + { + m_format_config = format_config; + } + + private: + C m_format_config; }; } diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index 2b06ffb17..979e59b05 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -183,6 +183,9 @@ namespace xt const std::string& path() const noexcept; void set_path(std::string& path); + template + void configure_format(C& config); + void flush(); private: @@ -209,7 +212,7 @@ namespace xt : m_value(value), m_dirty(dirty) { } - + template template inline auto xfile_reference::operator=(const V& v) -> self_type& @@ -344,7 +347,7 @@ namespace xt , m_path(path) { } - + template template inline auto xfile_array_container::operator=(const xexpression& e) -> self_type& @@ -357,13 +360,13 @@ namespace xt { return m_storage.size(); } - + template inline auto xfile_array_container::shape() const noexcept -> const shape_type& { return m_storage.shape(); } - + template inline auto xfile_array_container::layout() const noexcept -> layout_type { @@ -399,7 +402,7 @@ namespace xt m_storage.resize(std::forward(shape), strides); m_dirty = true; } - + template template inline auto xfile_array_container::reshape(S&& shape, layout_type layout) & -> self_type& @@ -532,17 +535,19 @@ namespace xt return m_storage.load_simd(i); } - - - - - template inline const std::string& xfile_array_container::path() const noexcept { return m_path; } + template + template + inline void xfile_array_container::configure_format(C& config) + { + m_io_handler.configure_format(config); + } + template inline void xfile_array_container::set_path(std::string& path) { diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 492962c88..164258d50 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -14,6 +14,7 @@ #include "xtensor/xchunk_store_manager.hpp" #include "xtensor/xfile_array.hpp" #include "xtensor/xdisk_io_handler.hpp" +#include "xtensor/xcsv.hpp" namespace xt { @@ -115,7 +116,7 @@ namespace xt { std::vector shape = {4, 4}; std::vector chunk_shape = {2, 2}; - xchunked_array>>> a1(shape, chunk_shape); + xchunked_array>>> a1(shape, chunk_shape); a1.chunks().set_pool_size(2); std::vector idx = {1, 2}; double v1 = 3.4; @@ -154,7 +155,7 @@ namespace xt TEST(xfile_array, indexed_access) { std::vector shape = {2, 2, 2}; - xfile_array> a; + xfile_array> a; a.resize(shape); double val = 3.; for (auto it: a) @@ -166,14 +167,14 @@ namespace xt TEST(xfile_array, assign_expression) { double v1 = 3.; - auto a1 = xfile_array>(broadcast(v1, {2, 2}), "a1"); + auto a1 = xfile_array>(broadcast(v1, {2, 2}), "a1"); for (const auto& v: a1) { EXPECT_EQ(v, v1); } double v2 = 2. * v1; - auto a2 = xfile_array>(a1 + a1, "a2"); + auto a2 = xfile_array>(a1 + a1, "a2"); for (const auto& v: a2) { EXPECT_EQ(v, v2); From 881e71b7163ab152a3b415a7162ddba83a16cedd Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 20 Aug 2020 19:09:41 +0200 Subject: [PATCH 074/606] Add xchunked_array extension template --- include/xtensor/xchunked_array.hpp | 133 +++++++++++++++-------------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 7deb6a342..624ec8f83 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -15,33 +15,36 @@ namespace xt * xchunked_array declaration * ******************************/ - template + class empty_extension {}; + + template class xchunked_array; - template - struct xcontainer_inner_types> + template + struct xcontainer_inner_types> { using chunk_type = typename chunk_storage::value_type; using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; using size_type = std::size_t; using storage_type = chunk_type; - using temporary_type = xchunked_array; + using temporary_type = xchunked_array; }; - template - struct xiterable_inner_types> + template + struct xiterable_inner_types> { using chunk_type = typename chunk_storage::value_type; using inner_shape_type = typename chunk_type::shape_type; - using const_stepper = xindexed_stepper, true>; - using stepper = xindexed_stepper, false>; + using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; }; - template - class xchunked_array: public xaccessible>, - public xiterable>, - public xcontainer_semantic> + template + class xchunked_array: public xaccessible>, + public xiterable>, + public xcontainer_semantic>, + public extension { public: @@ -49,7 +52,7 @@ namespace xt using chunk_type = typename chunk_storage::value_type; using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; - using self_type = xchunked_array; + using self_type = xchunked_array; using semantic_base = xcontainer_semantic; using iterable_base = xconst_iterable; using const_stepper = typename iterable_base::const_stepper; @@ -202,23 +205,23 @@ namespace xt * xchunked_array implementation * *********************************/ - template + template template - inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape) + inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape) { resize(std::forward(shape), std::forward(chunk_shape)); } - template + template template - inline xchunked_array::xchunked_array(const xexpression& e) + inline xchunked_array::xchunked_array(const xexpression& e) : xchunked_array(e, detail::chunk_helper::chunk_shape(e)) { } - template + template template - inline xchunked_array::xchunked_array(const xexpression& e, S&& chunk_shape) + inline xchunked_array::xchunked_array(const xexpression& e, S&& chunk_shape) { resize(e.derived_cast().shape(), std::forward(chunk_shape)); xstrided_slice_vector sv(m_chunk_shape.size()); // element slice corresponding to chunk @@ -262,127 +265,127 @@ namespace xt } } - template + template template - inline auto xchunked_array::operator=(const xexpression& e) -> self_type& + inline auto xchunked_array::operator=(const xexpression& e) -> self_type& { return semantic_base::operator=(e); } - template - inline auto xchunked_array::shape() const noexcept -> const shape_type& + template + inline auto xchunked_array::shape() const noexcept -> const shape_type& { return m_shape; } - template - inline auto xchunked_array::layout() const noexcept -> layout_type + template + inline auto xchunked_array::layout() const noexcept -> layout_type { return static_layout; } - template - inline bool xchunked_array::is_contiguous() const noexcept + template + inline bool xchunked_array::is_contiguous() const noexcept { return false; } - - template + + template template - inline auto xchunked_array::operator()(Idxs... idxs) -> reference + inline auto xchunked_array::operator()(Idxs... idxs) -> reference { auto ii = get_indexes(idxs...); auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); return chunk.element(ii.second.cbegin(), ii.second.cend()); } - template + template template - inline auto xchunked_array::operator()(Idxs... idxs) const -> const_reference + inline auto xchunked_array::operator()(Idxs... idxs) const -> const_reference { auto ii = get_indexes(idxs...); auto& chunk = m_chunks.element(ii.first.cbegin(), ii.first.cend()); return chunk.element(ii.second.cbegin(), ii.second.cend()); } - template + template template - inline auto xchunked_array::element(It first, It last) -> reference + inline auto xchunked_array::element(It first, It last) -> reference { auto ii = get_indexes_dynamic(first, last); auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); return chunk.element(ii.second.begin(), ii.second.end()); } - template + template template - inline auto xchunked_array::element(It first, It last) const -> const_reference + inline auto xchunked_array::element(It first, It last) const -> const_reference { auto ii = get_indexes_dynamic(first, last); auto& chunk = m_chunks.element(ii.first.begin(), ii.first.end()); return chunk.element(ii.second.begin(), ii.second.end()); } - template + template template - inline bool xchunked_array::broadcast_shape(S& s, bool) const + inline bool xchunked_array::broadcast_shape(S& s, bool) const { return xt::broadcast_shape(shape(), s); } - template + template template - inline auto xchunked_array::stepper_begin(const S& shape) noexcept -> stepper + inline auto xchunked_array::stepper_begin(const S& shape) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); return stepper(this, offset); } - template + template template - inline auto xchunked_array::stepper_end(const S& shape, layout_type) noexcept -> stepper + inline auto xchunked_array::stepper_end(const S& shape, layout_type) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); return stepper(this, offset, true); } - template + template template - inline auto xchunked_array::stepper_begin(const S& shape) const noexcept -> const_stepper + inline auto xchunked_array::stepper_begin(const S& shape) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); return const_stepper(this, offset); } - template + template template - inline auto xchunked_array::stepper_end(const S& shape, layout_type) const noexcept -> const_stepper + inline auto xchunked_array::stepper_end(const S& shape, layout_type) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); return const_stepper(this, offset, true); } - - template - inline auto xchunked_array::chunks() -> chunk_storage_type& + + template + inline auto xchunked_array::chunks() -> chunk_storage_type& { return m_chunks; } - template - inline auto xchunked_array::chunks() const -> const chunk_storage_type& + template + inline auto xchunked_array::chunks() const -> const chunk_storage_type& { return m_chunks; } - template - inline auto xchunked_array::chunk_shape() const -> const shape_type& + template + inline auto xchunked_array::chunk_shape() const -> const shape_type& { return m_chunk_shape; } - template + template template - inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) + inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) { // compute chunk number in each dimension (shape_of_chunks) std::vector shape_of_chunks(shape.size()); @@ -412,35 +415,35 @@ namespace xt m_chunk_shape = xtl::forward_sequence(chunk_shape); } - template + template template - inline auto xchunked_array::get_indexes(Idxs... idxs) const -> indexes_type + inline auto xchunked_array::get_indexes(Idxs... idxs) const -> indexes_type { auto chunk_indexes_packed = get_chunk_indexes(std::make_index_sequence(), idxs...); return unpack(chunk_indexes_packed); } - template + template template - inline std::pair xchunked_array::get_chunk_indexes_in_dimension(size_t dim, Idx idx) const + inline std::pair xchunked_array::get_chunk_indexes_in_dimension(size_t dim, Idx idx) const { size_t index_of_chunk = idx / m_chunk_shape[dim]; size_t index_in_chunk = idx - index_of_chunk * m_chunk_shape[dim]; return std::make_pair(index_of_chunk, index_in_chunk); } - template + template template - inline auto xchunked_array::get_chunk_indexes(std::index_sequence, Idxs... idxs) const + inline auto xchunked_array::get_chunk_indexes(std::index_sequence, Idxs... idxs) const -> chunk_indexes_type { chunk_indexes_type chunk_indexes = {{get_chunk_indexes_in_dimension(dims, idxs)...}}; return chunk_indexes; } - template + template template - inline auto xchunked_array::unpack(const std::array &arr) const -> static_indexes_type + inline auto xchunked_array::unpack(const std::array &arr) const -> static_indexes_type { std::array arr0; std::array arr1; @@ -452,9 +455,9 @@ namespace xt return std::make_pair(arr0, arr1); } - template + template template - inline auto xchunked_array::get_indexes_dynamic(It first, It last) const -> dynamic_indexes_type + inline auto xchunked_array::get_indexes_dynamic(It first, It last) const -> dynamic_indexes_type { auto size = static_cast(std::distance(first, last)); std::vector indexes_of_chunk(size); From 7f1bd68945a7cce64859ff02119e3761a126aa3f Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Mon, 24 Aug 2020 17:37:59 +0200 Subject: [PATCH 075/606] Refactored xdisk_io_handler --- include/xtensor/xcsv.hpp | 4 +- include/xtensor/xdisk_io_handler.hpp | 71 ++++++++++++++++++---------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/include/xtensor/xcsv.hpp b/include/xtensor/xcsv.hpp index 96de5a1f3..ceb8d6b3f 100644 --- a/include/xtensor/xcsv.hpp +++ b/include/xtensor/xcsv.hpp @@ -216,9 +216,9 @@ namespace xt }; template - auto load_file(std::istream& stream, const xcsv_config& config) + void load_file(std::istream& stream, xexpression& e, const xcsv_config& config) { - return load_csv(stream, config.delimiter, config.skip_rows, config.max_rows, config.comments); + e.derived_cast() = load_csv(stream, config.delimiter, config.skip_rows, config.max_rows, config.comments); } template diff --git a/include/xtensor/xdisk_io_handler.hpp b/include/xtensor/xdisk_io_handler.hpp index 9991166d0..2d4562529 100644 --- a/include/xtensor/xdisk_io_handler.hpp +++ b/include/xtensor/xdisk_io_handler.hpp @@ -10,46 +10,65 @@ namespace xt class xdisk_io_handler { public: + template - void write(xexpression& expression, std::string& path) + void write(const xexpression& expression, const std::string& path) const; + + template + void read(ET& array, const std::string& path, bool throw_on_fail = false) const; + + void configure_format(const C& format_config); + + private: + + C m_format_config; + }; + + template + template + inline void xdisk_io_handler::write(const xexpression& expression, const std::string& path) const + { + std::ofstream out_file(path, std::ofstream::binary); + if (out_file.is_open()) { - std::ofstream m_out_file(path, std::ofstream::binary); - if (m_out_file.is_open()) - { - dump_file(m_out_file, expression, m_format_config); - } - else - { - std::runtime_error("write: failed to open file " + path); - } + dump_file(out_file, expression, m_format_config); + } + else + { + std::runtime_error("write: failed to open file " + path); } + } - template - void read(ET& array, std::string& path) + template + template + inline void xdisk_io_handler::read(ET& array, const std::string& path, bool throw_on_fail) const + { + std::ifstream in_file(path, std::ifstream::binary); + if (in_file.is_open()) + { + load_file(in_file, array, m_format_config); + } + else { - // not all formats store the shape (e.g. Blosc) - // so we reshape after loading - std::ifstream m_in_file(path, std::ifstream::binary); - const auto shape = array.shape(); - if (m_in_file.is_open()) + if (throw_on_fail) { - array = load_file(m_in_file, m_format_config); - array.reshape(shape); + XTENSOR_THROW(std::runtime_error, "read: failed to open file " + path); } else { + auto shape = array.shape(); array = zeros(shape); } } + } + + template + inline void xdisk_io_handler::configure_format(const C& format_config) + { + m_format_config = format_config; + } - void configure_format(C& format_config) - { - m_format_config = format_config; - } - private: - C m_format_config; - }; } #endif From ad0f0a9b122f03f9787b2063ec1c1faa636798d5 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 27 Aug 2020 15:26:55 +0200 Subject: [PATCH 076/606] Fix exception for file write operation --- include/xtensor/xdisk_io_handler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xdisk_io_handler.hpp b/include/xtensor/xdisk_io_handler.hpp index 2d4562529..47095f777 100644 --- a/include/xtensor/xdisk_io_handler.hpp +++ b/include/xtensor/xdisk_io_handler.hpp @@ -35,7 +35,7 @@ namespace xt } else { - std::runtime_error("write: failed to open file " + path); + XTENSOR_THROW(std::runtime_error, "write: failed to open file " + path); } } From 757788642ef997f138e5975480f2562484f2c9e1 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 27 Aug 2020 16:03:38 +0200 Subject: [PATCH 077/606] Fix for empty file path --- include/xtensor/xfile_array.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index 979e59b05..2026338fa 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -554,7 +554,7 @@ namespace xt if (path != m_path) { // maybe write to old file - if (m_dirty) + if (m_dirty && !m_path.empty()) { m_io_handler.write(m_storage, m_path); m_dirty = false; @@ -568,7 +568,7 @@ namespace xt template inline void xfile_array_container::flush() { - if (m_dirty) + if (m_dirty && !m_path.empty()) { m_io_handler.write(m_storage, m_path); m_dirty = false; From 45eca8026092bd86ba19cf97559437e344439c11 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 27 Aug 2020 17:01:14 +0200 Subject: [PATCH 078/606] Add xfile_array.ignore_empty_path(bool) --- include/xtensor/xchunk_store_manager.hpp | 2 + include/xtensor/xfile_array.hpp | 48 +++++++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 2b185eae5..2fba2c001 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -131,6 +131,7 @@ namespace xt , m_index_pool(1u) , m_unload_index(0u) { + m_chunk_pool[0].ignore_empty_path(true); } template @@ -222,6 +223,7 @@ namespace xt for (auto& chunk: m_chunk_pool) { chunk.resize(chunk_shape); + chunk.ignore_empty_path(true); } } diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index 2026338fa..e38bc7586 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -181,6 +181,7 @@ namespace xt load_simd(size_type i) const; const std::string& path() const noexcept; + void ignore_empty_path(bool ignore); void set_path(std::string& path); template @@ -190,10 +191,13 @@ namespace xt private: + bool enable_io(std::string& path); + E m_storage; bool m_dirty; IOH m_io_handler; std::string m_path; + bool m_ignore_empty_path; }; template ::path(e)) + , m_ignore_empty_path(false) { } @@ -345,6 +350,7 @@ namespace xt , m_dirty(true) , m_io_handler() , m_path(path) + , m_ignore_empty_path(false) { } @@ -512,7 +518,7 @@ namespace xt { return reference(m_storage.data_element(i), m_dirty); } - + template inline auto xfile_array_container::data_element(size_type i) const -> const_reference { @@ -548,29 +554,59 @@ namespace xt m_io_handler.configure_format(config); } + template + inline void xfile_array_container::ignore_empty_path(bool ignore) + { + m_ignore_empty_path = ignore; + } + + template + inline bool xfile_array_container::enable_io(std::string& path) + { + bool res; + if (path.empty()) + { + res = !m_ignore_empty_path; + } + else + { + res = true; + } + return res; + } + template inline void xfile_array_container::set_path(std::string& path) { if (path != m_path) { // maybe write to old file - if (m_dirty && !m_path.empty()) + if (m_dirty) { - m_io_handler.write(m_storage, m_path); + if (enable_io(m_path)) + { + m_io_handler.write(m_storage, m_path); + } m_dirty = false; } m_path = path; // read new file - m_io_handler.read(m_storage, path); + if (enable_io(path)) + { + m_io_handler.read(m_storage, path); + } } } template inline void xfile_array_container::flush() { - if (m_dirty && !m_path.empty()) + if (m_dirty) { - m_io_handler.write(m_storage, m_path); + if (enable_io(m_path)) + { + m_io_handler.write(m_storage, m_path); + } m_dirty = false; } } From 216b03d65b2f7a3fa7b82c2690b5737e95e3e1a0 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 27 Aug 2020 17:13:21 +0200 Subject: [PATCH 079/606] Fix tests --- test/test_xchunked_array.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 164258d50..50fa8c08b 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -156,6 +156,7 @@ namespace xt { std::vector shape = {2, 2, 2}; xfile_array> a; + a.ignore_empty_path(true); a.resize(shape); double val = 3.; for (auto it: a) @@ -168,6 +169,7 @@ namespace xt { double v1 = 3.; auto a1 = xfile_array>(broadcast(v1, {2, 2}), "a1"); + a1.ignore_empty_path(true); for (const auto& v: a1) { EXPECT_EQ(v, v1); @@ -175,6 +177,7 @@ namespace xt double v2 = 2. * v1; auto a2 = xfile_array>(a1 + a1, "a2"); + a2.ignore_empty_path(true); for (const auto& v: a2) { EXPECT_EQ(v, v2); From 6cf63fa7761957fc7b3f5cad6589c760840b410a Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 27 Aug 2020 17:34:56 +0200 Subject: [PATCH 080/606] Changes according to review --- include/xtensor/xfile_array.hpp | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index e38bc7586..aa491dc76 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -191,7 +191,7 @@ namespace xt private: - bool enable_io(std::string& path); + bool enable_io(const std::string& path) const; E m_storage; bool m_dirty; @@ -561,18 +561,9 @@ namespace xt } template - inline bool xfile_array_container::enable_io(std::string& path) + inline bool xfile_array_container::enable_io(const std::string& path) const { - bool res; - if (path.empty()) - { - res = !m_ignore_empty_path; - } - else - { - res = true; - } - return res; + return !path.empty() || !m_ignore_empty_path; } template @@ -581,14 +572,7 @@ namespace xt if (path != m_path) { // maybe write to old file - if (m_dirty) - { - if (enable_io(m_path)) - { - m_io_handler.write(m_storage, m_path); - } - m_dirty = false; - } + flush(); m_path = path; // read new file if (enable_io(path)) From 499063f357e8ef9488049264c41d52e868a7642d Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Sun, 30 Aug 2020 23:51:41 +0200 Subject: [PATCH 081/606] Upgraded to xtl 0.6.16 --- .azure-pipelines/azure-pipelines-win.yml | 2 +- CMakeLists.txt | 2 +- README.md | 2 +- environment-dev.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 1a00afdc8..4d5bf66d1 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -37,7 +37,7 @@ jobs: conda install cmake==3.14.0 ^ ninja ^ nlohmann_json ^ - xtl==0.6.12 ^ + xtl==0.6.16 ^ xsimd==7.4.8 ^ python=3.6 conda list diff --git a/CMakeLists.txt b/CMakeLists.txt index 25574220a..1801fb65b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ message(STATUS "Building xtensor v${${PROJECT_NAME}_VERSION}") # Dependencies # ============ -set(xtl_REQUIRED_VERSION 0.6.9) +set(xtl_REQUIRED_VERSION 0.6.16) if(TARGET xtl) set(xtl_VERSION ${XTL_VERSION_MAJOR}.${XTL_VERSION_MINOR}.${XTL_VERSION_PATCH}) # Note: This is not SEMVER compatible comparison diff --git a/README.md b/README.md index bfb1889be..ac34b6cf1 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| -| master | ^0.6.15 | ^7.4.8 | +| master | ^0.6.16 | ^7.4.8 | | 0.21.5 | ^0.6.12 | ^7.4.6 | | 0.21.4 | ^0.6.12 | ^7.4.6 | | 0.21.3 | ^0.6.9 | ^7.4.4 | diff --git a/environment-dev.yml b/environment-dev.yml index 1393e6ba5..3a0c72674 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -3,6 +3,6 @@ channels: - conda-forge dependencies: - cmake - - xtl=0.6.15 + - xtl=0.6.16 - xsimd=7.4.8 - nlohmann_json From 9dfcfb4fac04535418b8ddebd90fe302dfbde5f2 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Mon, 31 Aug 2020 12:04:04 +0200 Subject: [PATCH 082/606] Implemented zarray --- .azure-pipelines/azure-pipelines-win.yml | 2 +- CMakeLists.txt | 1 + environment-dev.yml | 2 +- include/xtensor/zarray.hpp | 342 +++++++++++++++++++++++ test/CMakeLists.txt | 1 + test/test_zarray.cpp | 25 ++ 6 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 include/xtensor/zarray.hpp create mode 100644 test/test_zarray.cpp diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 4d5bf66d1..3c05a6e12 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -37,7 +37,7 @@ jobs: conda install cmake==3.14.0 ^ ninja ^ nlohmann_json ^ - xtl==0.6.16 ^ + xtl==0.6.17 ^ xsimd==7.4.8 ^ python=3.6 conda list diff --git a/CMakeLists.txt b/CMakeLists.txt index 1801fb65b..ea389080e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,6 +179,7 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xvectorize.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xview.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xview_utils.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/zarray.hpp ) add_library(xtensor INTERFACE) diff --git a/environment-dev.yml b/environment-dev.yml index 3a0c72674..e5a43cb3f 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -3,6 +3,6 @@ channels: - conda-forge dependencies: - cmake - - xtl=0.6.16 + - xtl=0.6.17 - xsimd=7.4.8 - nlohmann_json diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp new file mode 100644 index 000000000..2db8318ee --- /dev/null +++ b/include/xtensor/zarray.hpp @@ -0,0 +1,342 @@ +#ifndef XTENSOR_ZARRAY_HPP +#define XTENSOR_ZARRAY_HPP + +#include + +#include + +#include "xarray.hpp" + +namespace xt +{ + + class zarray_impl; + + /********** + * zarray * + **********/ + + class zarray + { + public: + + using implementation_ptr = std::unique_ptr; + + zarray() = default; + ~zarray() = default; + + template + zarray(E&& e); + + zarray(implementation_ptr&& impl); + + zarray(const zarray& rhs); + zarray& operator=(const zarray& rhs); + + zarray(zarray&& rhs); + zarray& operator=(zarray&& rhs); + + void swap(zarray& rhs); + + zarray_impl& get_implementation(); + const zarray_impl& get_implementation() const; + + template + xarray& get_array(); + + template + const xarray& get_array() const; + + private: + + implementation_ptr p_impl; + }; + + /*************** + * zarray_impl * + ***************/ + + class zarray_impl + { + public: + + using self_type = zarray_impl; + + virtual ~zarray_impl() = default; + + zarray_impl(zarray_impl&&) = delete; + zarray_impl& operator=(const zarray_impl&) = delete; + zarray_impl& operator=(zarray_impl&&) = delete; + + virtual self_type* clone() const = 0; + + XTL_IMPLEMENT_INDEXABLE_CLASS() + + protected: + + zarray_impl() = default; + zarray_impl(const zarray_impl&) = default; + }; + + /**************** + * ztyped_array * + ****************/ + + template + class ztyped_array : public zarray_impl + { + public: + + virtual ~ztyped_array() = default; + + virtual xarray& get_array() = 0; + virtual const xarray& get_array() const = 0; + + XTL_IMPLEMENT_INDEXABLE_CLASS() + + protected: + + ztyped_array() = default; + ztyped_array(const ztyped_array&) = default; + }; + + /*********************** + * zexpression_wrapper * + ***********************/ + + template + class zexpression_wrapper : public ztyped_array::value_type> + { + public: + + using self_type = zexpression_wrapper; + using value_type = typename std::decay_t::value_type; + using base_type = ztyped_array; + + template + zexpression_wrapper(E&& e); + + virtual ~zexpression_wrapper() = default; + + xarray& get_array() override; + const xarray& get_array() const override; + + self_type* clone() const override; + + private: + + zexpression_wrapper(const zexpression_wrapper&) = default; + + void compute_cache() const; + + CTE m_expression; + mutable xarray m_cache; + mutable bool m_cache_initialized; + }; + + /****************** + * zarray_wrapper * + ******************/ + + template + class zarray_wrapper : public ztyped_array::value_type> + { + public: + + using self_type = zarray_wrapper; + using value_type = typename std::decay_t::value_type; + using base_type = ztyped_array; + + template + zarray_wrapper(E&& e); + + virtual ~zarray_wrapper() = default; + + xarray& get_array() override; + const xarray& get_array() const override; + + self_type* clone() const override; + + private: + + zarray_wrapper(const zarray_wrapper&) = default; + + CTE m_array; + }; + + /************************* + * zarray implementation * + *************************/ + + namespace detail + { + template + struct is_xarray : std::false_type + { + }; + + template + struct is_xarray> : std::true_type + { + }; + + template + struct zwrapper_builder + { + using closure_type = xtl::closure_type_t; + using wrapper_type = std::conditional_t>::value, + zarray_wrapper, + zexpression_wrapper>; + + template + static wrapper_type* run(OE&& e) + { + return new wrapper_type(std::forward(e)); + } + }; + + template + inline auto build_zarray(E&& e) + { + return zwrapper_builder::run(std::forward(e)); + } + } + + template + inline zarray::zarray(E&& e) + : p_impl(detail::build_zarray(std::forward(e))) + { + } + + inline zarray::zarray(implementation_ptr&& impl) + : p_impl(std::move(impl)) + { + } + + inline zarray::zarray(const zarray& rhs) + : p_impl(rhs.p_impl->clone()) + { + } + + inline zarray& zarray::operator=(const zarray& rhs) + { + zarray tmp(rhs); + swap(tmp); + return *this; + } + + inline zarray::zarray(zarray&& rhs) + : p_impl(std::move(rhs.p_impl)) + { + } + + inline zarray& zarray::operator=(zarray&& rhs) + { + swap(rhs); + return *this; + } + + inline void zarray::swap(zarray& rhs) + { + std::swap(p_impl, rhs.p_impl); + } + + inline zarray_impl& zarray::get_implementation() + { + return *p_impl; + } + + inline const zarray_impl& zarray::get_implementation() const + { + return *p_impl; + } + + template + inline xarray& zarray::get_array() + { + return dynamic_cast*>(p_impl.get())->get_array(); + } + + template + inline const xarray& zarray::get_array() const + { + return dynamic_cast*>(p_impl.get())->get_array(); + } + + /*********************** + * zexpression_wrapper * + ***********************/ + + template + template + inline zexpression_wrapper::zexpression_wrapper(E&& e) + : base_type() + , m_expression(std::forward(e)) + , m_cache() + , m_cache_initialized(false) + { + } + + template + inline auto zexpression_wrapper::get_array() -> xarray& + { + compute_cache(); + return m_cache; + } + + template + inline auto zexpression_wrapper::get_array() const -> const xarray& + { + compute_cache(); + return m_cache; + } + + template + inline auto zexpression_wrapper::clone() const -> self_type* + { + return new self_type(*this); + } + + template + inline void zexpression_wrapper::compute_cache() const + { + if (!m_cache_initialized) + { + m_cache = m_expression; + m_cache_initialized = true; + } + } + + /****************** + * zarray_wrapper * + ******************/ + + template + template + inline zarray_wrapper::zarray_wrapper(E&& e) + : base_type() + , m_array(std::forward(e)) + { + } + + template + inline auto zarray_wrapper::get_array() -> xarray& + { + return m_array; + } + + template + inline auto zarray_wrapper::get_array() const -> const xarray& + { + return m_array; + } + + template + inline auto zarray_wrapper::clone() const -> self_type* + { + return new self_type(*this); + } +} + +#endif + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 57d935c8b..e64618b14 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -181,6 +181,7 @@ set(COMMON_BASE test_xview.cpp test_xview_semantic.cpp test_xutils.cpp + test_zarray.cpp ) set(XTENSOR_TESTS diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp new file mode 100644 index 000000000..07be3e947 --- /dev/null +++ b/test/test_zarray.cpp @@ -0,0 +1,25 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include "gtest/gtest.h" +#include "xtensor/zarray.hpp" + +namespace xt +{ + TEST(zarray, value_semantics) + { + xarray a = {{1., 2.}, {3., 4.}}; + xarray ra = {{2., 2.}, {3., 4.}}; + zarray da(a); + da.get_array()(0, 0) = 2.; + + EXPECT_EQ(a, ra); + } +} + From 2007513ed2781bf8d6682607328aafbb4d36bbd7 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 1 Sep 2020 11:34:41 +0200 Subject: [PATCH 083/606] Skeleton of the dynamic expression system --- CMakeLists.txt | 4 + include/xtensor/zarray.hpp | 232 ++---------------------------- include/xtensor/zarray_impl.hpp | 244 ++++++++++++++++++++++++++++++++ include/xtensor/zdispatcher.hpp | 85 +++++++++++ include/xtensor/zfunction.hpp | 18 +++ include/xtensor/zmath.hpp | 46 ++++++ test/test_zarray.cpp | 20 +++ 7 files changed, 427 insertions(+), 222 deletions(-) create mode 100644 include/xtensor/zarray_impl.hpp create mode 100644 include/xtensor/zdispatcher.hpp create mode 100644 include/xtensor/zfunction.hpp create mode 100644 include/xtensor/zmath.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ea389080e..82581339c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,10 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xview.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xview_utils.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zarray.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/zarray_impl.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/zdispatcher.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/zfunction.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/zmath.hpp ) add_library(xtensor INTERFACE) diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 2db8318ee..364fe2afa 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -1,3 +1,12 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + #ifndef XTENSOR_ZARRAY_HPP #define XTENSOR_ZARRAY_HPP @@ -6,12 +15,11 @@ #include #include "xarray.hpp" +#include "zarray_impl.hpp" namespace xt { - class zarray_impl; - /********** * zarray * **********/ @@ -52,156 +60,10 @@ namespace xt implementation_ptr p_impl; }; - /*************** - * zarray_impl * - ***************/ - - class zarray_impl - { - public: - - using self_type = zarray_impl; - - virtual ~zarray_impl() = default; - - zarray_impl(zarray_impl&&) = delete; - zarray_impl& operator=(const zarray_impl&) = delete; - zarray_impl& operator=(zarray_impl&&) = delete; - - virtual self_type* clone() const = 0; - - XTL_IMPLEMENT_INDEXABLE_CLASS() - - protected: - - zarray_impl() = default; - zarray_impl(const zarray_impl&) = default; - }; - - /**************** - * ztyped_array * - ****************/ - - template - class ztyped_array : public zarray_impl - { - public: - - virtual ~ztyped_array() = default; - - virtual xarray& get_array() = 0; - virtual const xarray& get_array() const = 0; - - XTL_IMPLEMENT_INDEXABLE_CLASS() - - protected: - - ztyped_array() = default; - ztyped_array(const ztyped_array&) = default; - }; - - /*********************** - * zexpression_wrapper * - ***********************/ - - template - class zexpression_wrapper : public ztyped_array::value_type> - { - public: - - using self_type = zexpression_wrapper; - using value_type = typename std::decay_t::value_type; - using base_type = ztyped_array; - - template - zexpression_wrapper(E&& e); - - virtual ~zexpression_wrapper() = default; - - xarray& get_array() override; - const xarray& get_array() const override; - - self_type* clone() const override; - - private: - - zexpression_wrapper(const zexpression_wrapper&) = default; - - void compute_cache() const; - - CTE m_expression; - mutable xarray m_cache; - mutable bool m_cache_initialized; - }; - - /****************** - * zarray_wrapper * - ******************/ - - template - class zarray_wrapper : public ztyped_array::value_type> - { - public: - - using self_type = zarray_wrapper; - using value_type = typename std::decay_t::value_type; - using base_type = ztyped_array; - - template - zarray_wrapper(E&& e); - - virtual ~zarray_wrapper() = default; - - xarray& get_array() override; - const xarray& get_array() const override; - - self_type* clone() const override; - - private: - - zarray_wrapper(const zarray_wrapper&) = default; - - CTE m_array; - }; - /************************* * zarray implementation * *************************/ - namespace detail - { - template - struct is_xarray : std::false_type - { - }; - - template - struct is_xarray> : std::true_type - { - }; - - template - struct zwrapper_builder - { - using closure_type = xtl::closure_type_t; - using wrapper_type = std::conditional_t>::value, - zarray_wrapper, - zexpression_wrapper>; - - template - static wrapper_type* run(OE&& e) - { - return new wrapper_type(std::forward(e)); - } - }; - - template - inline auto build_zarray(E&& e) - { - return zwrapper_builder::run(std::forward(e)); - } - } - template inline zarray::zarray(E&& e) : p_impl(detail::build_zarray(std::forward(e))) @@ -262,80 +124,6 @@ namespace xt { return dynamic_cast*>(p_impl.get())->get_array(); } - - /*********************** - * zexpression_wrapper * - ***********************/ - - template - template - inline zexpression_wrapper::zexpression_wrapper(E&& e) - : base_type() - , m_expression(std::forward(e)) - , m_cache() - , m_cache_initialized(false) - { - } - - template - inline auto zexpression_wrapper::get_array() -> xarray& - { - compute_cache(); - return m_cache; - } - - template - inline auto zexpression_wrapper::get_array() const -> const xarray& - { - compute_cache(); - return m_cache; - } - - template - inline auto zexpression_wrapper::clone() const -> self_type* - { - return new self_type(*this); - } - - template - inline void zexpression_wrapper::compute_cache() const - { - if (!m_cache_initialized) - { - m_cache = m_expression; - m_cache_initialized = true; - } - } - - /****************** - * zarray_wrapper * - ******************/ - - template - template - inline zarray_wrapper::zarray_wrapper(E&& e) - : base_type() - , m_array(std::forward(e)) - { - } - - template - inline auto zarray_wrapper::get_array() -> xarray& - { - return m_array; - } - - template - inline auto zarray_wrapper::get_array() const -> const xarray& - { - return m_array; - } - - template - inline auto zarray_wrapper::clone() const -> self_type* - { - return new self_type(*this); - } } #endif diff --git a/include/xtensor/zarray_impl.hpp b/include/xtensor/zarray_impl.hpp new file mode 100644 index 000000000..d2bd20d74 --- /dev/null +++ b/include/xtensor/zarray_impl.hpp @@ -0,0 +1,244 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XTENSOR_ZARRAY_IMPL_HPP +#define XTENSOR_ZARRAY_IMPL_HPP + +#include "xarray.hpp" + +namespace xt +{ + + /*************** + * zarray_impl * + ***************/ + + class zarray_impl + { + public: + + using self_type = zarray_impl; + + virtual ~zarray_impl() = default; + + zarray_impl(zarray_impl&&) = delete; + zarray_impl& operator=(const zarray_impl&) = delete; + zarray_impl& operator=(zarray_impl&&) = delete; + + virtual self_type* clone() const = 0; + + XTL_IMPLEMENT_INDEXABLE_CLASS() + + protected: + + zarray_impl() = default; + zarray_impl(const zarray_impl&) = default; + }; + + /**************** + * ztyped_array * + ****************/ + + template + class ztyped_array : public zarray_impl + { + public: + + virtual ~ztyped_array() = default; + + virtual xarray& get_array() = 0; + virtual const xarray& get_array() const = 0; + + XTL_IMPLEMENT_INDEXABLE_CLASS() + + protected: + + ztyped_array() = default; + ztyped_array(const ztyped_array&) = default; + }; + + /*********************** + * zexpression_wrapper * + ***********************/ + + template + class zexpression_wrapper : public ztyped_array::value_type> + { + public: + + using self_type = zexpression_wrapper; + using value_type = typename std::decay_t::value_type; + using base_type = ztyped_array; + + template + zexpression_wrapper(E&& e); + + virtual ~zexpression_wrapper() = default; + + xarray& get_array() override; + const xarray& get_array() const override; + + self_type* clone() const override; + + private: + + zexpression_wrapper(const zexpression_wrapper&) = default; + + void compute_cache() const; + + CTE m_expression; + mutable xarray m_cache; + mutable bool m_cache_initialized; + }; + + /****************** + * zarray_wrapper * + ******************/ + + template + class zarray_wrapper : public ztyped_array::value_type> + { + public: + + using self_type = zarray_wrapper; + using value_type = typename std::decay_t::value_type; + using base_type = ztyped_array; + + template + zarray_wrapper(E&& e); + + virtual ~zarray_wrapper() = default; + + xarray& get_array() override; + const xarray& get_array() const override; + + self_type* clone() const override; + + private: + + zarray_wrapper(const zarray_wrapper&) = default; + + CTE m_array; + }; + + /*********************** + * zexpression_wrapper * + ***********************/ + + template + template + inline zexpression_wrapper::zexpression_wrapper(E&& e) + : base_type() + , m_expression(std::forward(e)) + , m_cache() + , m_cache_initialized(false) + { + } + + template + inline auto zexpression_wrapper::get_array() -> xarray& + { + compute_cache(); + return m_cache; + } + + template + inline auto zexpression_wrapper::get_array() const -> const xarray& + { + compute_cache(); + return m_cache; + } + + template + inline auto zexpression_wrapper::clone() const -> self_type* + { + return new self_type(*this); + } + + template + inline void zexpression_wrapper::compute_cache() const + { + if (!m_cache_initialized) + { + m_cache = m_expression; + m_cache_initialized = true; + } + } + + /****************** + * zarray_wrapper * + ******************/ + + template + template + inline zarray_wrapper::zarray_wrapper(E&& e) + : base_type() + , m_array(std::forward(e)) + { + } + + template + inline auto zarray_wrapper::get_array() -> xarray& + { + return m_array; + } + + template + inline auto zarray_wrapper::get_array() const -> const xarray& + { + return m_array; + } + + template + inline auto zarray_wrapper::clone() const -> self_type* + { + return new self_type(*this); + } + + /****************** + * zarray builder * + ******************/ + + namespace detail + { + template + struct is_xarray : std::false_type + { + }; + + template + struct is_xarray> : std::true_type + { + }; + + template + struct zwrapper_builder + { + using closure_type = xtl::closure_type_t; + using wrapper_type = std::conditional_t>::value, + zarray_wrapper, + zexpression_wrapper>; + + template + static wrapper_type* run(OE&& e) + { + return new wrapper_type(std::forward(e)); + } + }; + + template + inline auto build_zarray(E&& e) + { + return zwrapper_builder::run(std::forward(e)); + } + } +} + +#endif + diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp new file mode 100644 index 000000000..d2ba4eb3d --- /dev/null +++ b/include/xtensor/zdispatcher.hpp @@ -0,0 +1,85 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef ZDISPATCHER_HPP +#define ZDISPATCHER_HPP + +#include + +#include "zmath.hpp" + +namespace xt +{ + namespace mpl = xtl::mpl; + + using supported_type = mpl::vector; + + template + using zdispatcher_impl = xtl::functor_dispatcher + < + type_list, + void, + xtl::static_caster, + xtl::basic_fast_dispatcher + >; + + using zsingle_dispatcher_impl = zdispatcher_impl + < + mpl::vector + >; + + using zdouble_dispatcher_impl = zdispatcher_impl + < + mpl::vector + >; + + template + struct zsingle_dispatcher_base + { + static zsingle_dispatcher_impl& get() + { + static zsingle_dispatcher_impl dispatcher; + return dispatcher; + } + + static void init() + { + D::template insert(); + D::template insert(); + } + + static void dispatch(const zarray_impl& z1, zarray_impl& res) + { + get().dispatch(z1, res); + } + }; + + template + class zsingle_dispatcher; + +#define DEFINE_SINGLE_DISPATCHER(FUNCTOR, FUNCTION)\ + template <> \ + struct zsingle_dispatcher\ + : private zsingle_dispatcher_base> \ + {\ + using base_type = zsingle_dispatcher_base>; \ + using base_type::dispatch; \ + using base_type::init; \ + template \ + static void insert() \ + {\ + base_type::get().template insert, ztyped_array>(&FUNCTION); \ + }\ + } + +DEFINE_SINGLE_DISPATCHER(math::exp_fun, zexp); + +} + +#endif diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp new file mode 100644 index 000000000..1d7069e9f --- /dev/null +++ b/include/xtensor/zfunction.hpp @@ -0,0 +1,18 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XTENSOR_ZFUNCTION_HPP +#define XTENSOR_ZFUNCTION_HPP + +namespace xt +{ +} + +#endif + diff --git a/include/xtensor/zmath.hpp b/include/xtensor/zmath.hpp new file mode 100644 index 000000000..6844816a1 --- /dev/null +++ b/include/xtensor/zmath.hpp @@ -0,0 +1,46 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XTENSOR_ZMATH_HPP +#define XTENSOR_ZMATH_HPP + +#include "xmath.hpp" +#include "zarray_impl.hpp" + +namespace xt +{ + namespace detail + { + // For further improvement: move shape computation + // at the beginning of a zarray assignment so it is computed + // only once + template + inline void zassign_data(xexpression& e1, const xexpression& e2) + { + e1.derived_cast() = e2.derived_cast(); + } + } + + template + inline void zadd(const ztyped_array& z1, + const ztyped_array& z2, + ztyped_array& zres) + { + detail::zassign_data(zres.get_array(), z1.get_array() + z2.get_array()); + } + + template + inline void zexp(const ztyped_array& z1, + ztyped_array& zres) + { + detail::zassign_data(zres.get_array(), xt::exp(z1.get_array())); + } +} + +#endif diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index 07be3e947..332b405f1 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -9,7 +9,9 @@ #include "gtest/gtest.h" #include "xtensor/zarray.hpp" +#include "xtensor/zdispatcher.hpp" +#ifndef XTENSOR_DISABLE_EXCEPTIONS namespace xt { TEST(zarray, value_semantics) @@ -21,5 +23,23 @@ namespace xt EXPECT_EQ(a, ra); } + + // TODO : move to dedicated test file + TEST(zarray, dispatching) + { + using dispatcher_type = zsingle_dispatcher; + dispatcher_type::init(); + + xarray a = {{0.5, 1.5}, {2.5, 3.5}}; + xarray expa = {{std::exp(0.5), std::exp(1.5)}, {std::exp(2.5), std::exp(3.5)}}; + xarray res; + zarray za(a); + zarray zres(res); + + dispatcher_type::dispatch(za.get_implementation(), zres.get_implementation()); + + EXPECT_EQ(expa, res); + } } +#endif From befb57c5df854e345cb4ea89de63b887629edc20 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 2 Sep 2020 09:30:35 +0200 Subject: [PATCH 084/606] Implemented zfunctions, equivalent of xfunction for dynamic expression system --- .azure-pipelines/azure-pipelines-win.yml | 2 +- environment-dev.yml | 2 +- include/xtensor/zdispatcher.hpp | 161 ++++++++++++++++++----- include/xtensor/zfunction.hpp | 62 +++++++++ include/xtensor/zmath.hpp | 68 ++++++++-- test/test_zarray.cpp | 34 ++++- 6 files changed, 280 insertions(+), 49 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 3c05a6e12..0637ae430 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -37,7 +37,7 @@ jobs: conda install cmake==3.14.0 ^ ninja ^ nlohmann_json ^ - xtl==0.6.17 ^ + xtl==0.6.18 ^ xsimd==7.4.8 ^ python=3.6 conda list diff --git a/environment-dev.yml b/environment-dev.yml index e5a43cb3f..b7bfd103e 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -3,6 +3,6 @@ channels: - conda-forge dependencies: - cmake - - xtl=0.6.17 + - xtl=0.6.18 - xsimd=7.4.8 - nlohmann_json diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp index d2ba4eb3d..c8b6e9b77 100644 --- a/include/xtensor/zdispatcher.hpp +++ b/include/xtensor/zdispatcher.hpp @@ -7,8 +7,8 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef ZDISPATCHER_HPP -#define ZDISPATCHER_HPP +#ifndef XTENSOR_ZDISPATCHER_HPP +#define XTENSOR_ZDISPATCHER_HPP #include @@ -21,7 +21,7 @@ namespace xt using supported_type = mpl::vector; template - using zdispatcher_impl = xtl::functor_dispatcher + using zrun_dispatcher_impl = xtl::functor_dispatcher < type_list, void, @@ -29,57 +29,150 @@ namespace xt xtl::basic_fast_dispatcher >; - using zsingle_dispatcher_impl = zdispatcher_impl + template + using ztype_dispatcher_impl = xtl::functor_dispatcher < - mpl::vector + type_list, + size_t, + xtl::static_caster, + xtl::basic_fast_dispatcher >; - using zdouble_dispatcher_impl = zdispatcher_impl - < - mpl::vector - >; + /********************** + * zdouble_dispatcher * + **********************/ + + // Double dispatchers are used for unary operations. + // They dispatch on the single argument and on the + // result. - template - struct zsingle_dispatcher_base + template + class zdouble_dispatcher { - static zsingle_dispatcher_impl& get() + private: + + using zfunctor_type = get_zmapped_functor_t; + using ztype_dispatcher = ztype_dispatcher_impl>; + using zrun_dispatcher = zrun_dispatcher_impl>; + + static ztype_dispatcher& type_dispatcher() { - static zsingle_dispatcher_impl dispatcher; + static ztype_dispatcher dispatcher; return dispatcher; } + static zrun_dispatcher& run_dispatcher() + { + static zrun_dispatcher dispatcher; + return dispatcher; + } + + public: + + template + static void insert() + { + using arg_type = const ztyped_array; + using res_type = ztyped_array; + run_dispatcher().template insert(&zfunctor_type::template run); + type_dispatcher().template insert(&zfunctor_type::template index); + } + static void init() { - D::template insert(); - D::template insert(); + insert(); + insert(); } static void dispatch(const zarray_impl& z1, zarray_impl& res) { - get().dispatch(z1, res); + run_dispatcher().dispatch(z1, res); + } + + static size_t get_type_index(const zarray_impl& z1) + { + return type_dispatcher().dispatch(z1); + } + }; + + /********************** + * ztriple_dispatcher * + **********************/ + + // Triple dispatchers are used for binary operations. + // They dispatch on both arguments and on the result. + + template + class ztriple_dispatcher + { + private: + + using zfunctor_type = get_zmapped_functor_t; + using ztype_dispatcher = ztype_dispatcher_impl>; + using zrun_dispatcher = zrun_dispatcher_impl>; + + static ztype_dispatcher& type_dispatcher() + { + static ztype_dispatcher dispatcher; + return dispatcher; + } + + static zrun_dispatcher& run_dispatcher() + { + static zrun_dispatcher dispatcher; + return dispatcher; + } + + public: + + template + static void insert() + { + using arg_type1 = const ztyped_array; + using arg_type2 = const ztyped_array; + using res_type = ztyped_array; + run_dispatcher().template insert(&zfunctor_type::template run); + type_dispatcher().template insert(&zfunctor_type::template index); + } + + static void init() + { + insert(); + insert(); + } + + static void dispatch(const zarray_impl& z1, const zarray_impl& z2, zarray_impl& res) + { + run_dispatcher().dispatch(z1, z2, res); + } + + static size_t get_type_index(const zarray_impl& z1, const zarray_impl& z2) + { + return type_dispatcher().dispatch(z1, z2); } }; + /*************** + * zdispatcher * + ***************/ + + template + struct zdispatcher; + template - class zsingle_dispatcher; - -#define DEFINE_SINGLE_DISPATCHER(FUNCTOR, FUNCTION)\ - template <> \ - struct zsingle_dispatcher\ - : private zsingle_dispatcher_base> \ - {\ - using base_type = zsingle_dispatcher_base>; \ - using base_type::dispatch; \ - using base_type::init; \ - template \ - static void insert() \ - {\ - base_type::get().template insert, ztyped_array>(&FUNCTION); \ - }\ - } - -DEFINE_SINGLE_DISPATCHER(math::exp_fun, zexp); + struct zdispatcher + { + using type = zdouble_dispatcher; + }; + + template + struct zdispatcher + { + using type = ztriple_dispatcher; + }; + template + using zdispatcher_t = typename zdispatcher::type; } #endif diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp index 1d7069e9f..82997e376 100644 --- a/include/xtensor/zfunction.hpp +++ b/include/xtensor/zfunction.hpp @@ -10,8 +10,70 @@ #ifndef XTENSOR_ZFUNCTION_HPP #define XTENSOR_ZFUNCTION_HPP +#include +#include + +#include "zdispatcher.hpp" + namespace xt { + template + class zfunction + { + public: + + using self_type = zfunction; + using tuple_type = std::tuple; + using functor_type = F; + + template , self_type>::value>> + zfunction(Func&& f, CTA&&... e) noexcept; + + zarray_impl& assign_to(zarray_impl& res) const; + + private: + + template + std::enable_if_t::value, const zarray_impl&> + get_array_impl(const E& e, zarray_impl& res) const + { + return e.assign_to(res); + } + + template + std::enable_if_t::value, const zarray_impl&> + get_array_impl(E& e, zarray_impl&) const + { + return e.get_implementation(); + } + + template + zarray_impl& assign_to_impl(std::index_sequence, zarray_impl& res) const; + + tuple_type m_e; + }; + + template + template + inline zfunction::zfunction(Func&&, CTA&&... e) noexcept + : m_e(std::forward(e)...) + { + } + + template + inline zarray_impl& zfunction::assign_to(zarray_impl& res) const + { + return assign_to_impl(std::make_index_sequence(), res); + } + + template + template + inline zarray_impl& zfunction::assign_to_impl(std::index_sequence, zarray_impl& res) const + { + // To do: call assign_to on zfunciton arguments + zdispatcher_t::dispatch(get_array_impl(std::get(m_e), res)..., res); + return res; + } } #endif diff --git a/include/xtensor/zmath.hpp b/include/xtensor/zmath.hpp index 6844816a1..0d1d0e758 100644 --- a/include/xtensor/zmath.hpp +++ b/include/xtensor/zmath.hpp @@ -27,20 +27,66 @@ namespace xt } } - template - inline void zadd(const ztyped_array& z1, - const ztyped_array& z2, - ztyped_array& zres) + template + struct get_zmapped_functor; + + template + using get_zmapped_functor_t = typename get_zmapped_functor::type; + +/*#define DEFINE_ZFUNCTOR_MAPPING(XF, ZF) \ + template <> \ + struct get_zmapped_functor \ + { using type = ZF; } +*/ + struct zadd { - detail::zassign_data(zres.get_array(), z1.get_array() + z2.get_array()); - } + template + static void run(const ztyped_array& z1, + const ztyped_array& z2, + ztyped_array& zres) + { + detail::zassign_data(zres.get_array(), z1.get_array() + z2.get_array()); + } - template - inline void zexp(const ztyped_array& z1, - ztyped_array& zres) + template + static size_t index(const ztyped_array&, const ztyped_array&) + { + using result_type = ztyped_array() + std::declval())>; + return result_type::get_class_static_index(); + } + }; + + template <> + struct get_zmapped_functor { - detail::zassign_data(zres.get_array(), xt::exp(z1.get_array())); - } + using type = zadd; + }; + + //DEFINE_ZFUNCTOR_MAPPING((detail::plus), zadd); + + struct zexp + { + template + static void run(const ztyped_array& z, + ztyped_array& zres) + { + detail::zassign_data(zres.get_array(), xt::exp(z.get_array())); + } + + template + static size_t index(const ztyped_array&) + { + using value_type = decltype(std::declval()(std::declval())); + return ztyped_array::get_class_static_index(); + } + }; + + template <> + struct get_zmapped_functor + { + using type = zexp; + }; + //DEFINE_ZFUNCTOR_MAPPING((math::exp_fun), zexp); } #endif diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index 332b405f1..4a77b51a2 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -9,7 +9,7 @@ #include "gtest/gtest.h" #include "xtensor/zarray.hpp" -#include "xtensor/zdispatcher.hpp" +#include "xtensor/zfunction.hpp" #ifndef XTENSOR_DISABLE_EXCEPTIONS namespace xt @@ -27,7 +27,7 @@ namespace xt // TODO : move to dedicated test file TEST(zarray, dispatching) { - using dispatcher_type = zsingle_dispatcher; + using dispatcher_type = zdispatcher_t; dispatcher_type::init(); xarray a = {{0.5, 1.5}, {2.5, 3.5}}; @@ -40,6 +40,36 @@ namespace xt EXPECT_EQ(expa, res); } + + // TODO: move to dedicated test file + TEST(zarray, add) + { + using exp_dispatcher_type = zdispatcher_t; + exp_dispatcher_type::init(); + + using add_dispatcher_type = zdispatcher_t; + add_dispatcher_type::init(); + + using nested_zfunction_type = zfunction; + using zfunction_type = zfunction; + + xarray a = {{0.5, 1.5}, {2.5, 3.5}}; + xarray b = {{-0.2, 2.4}, {1.3, 4.7}}; + xarray res; + + zarray za(a); + zarray zb(b); + zarray zres(res); + + zfunction_type f(zadd(), za, nested_zfunction_type(zexp(), zb)); + f.assign_to(zres.get_implementation()); + + auto expected = xarray::from_shape({2, 2}); + std::transform(a.cbegin(), a.cend(), b.cbegin(), expected.begin(), + [](const double& lhs, const double& rhs) { return lhs + std::exp(rhs); }); + + EXPECT_TRUE(all(isclose(res, expected))); + } } #endif From 9489dd008c21d9cf67fc2ccaef7bcc0ec0381d8a Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 3 Sep 2020 15:03:31 +0200 Subject: [PATCH 085/606] Implemented allocate_result in zfunction --- include/xtensor/zdispatcher.hpp | 51 +++++++++++++++++++++++++++ include/xtensor/zfunction.hpp | 62 +++++++++++++++++++++++++++++++-- test/test_zarray.cpp | 3 ++ 3 files changed, 114 insertions(+), 2 deletions(-) diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp index c8b6e9b77..3fd3f5fed 100644 --- a/include/xtensor/zdispatcher.hpp +++ b/include/xtensor/zdispatcher.hpp @@ -173,6 +173,57 @@ namespace xt template using zdispatcher_t = typename zdispatcher::type; + + /************************ + * zarray_impl_register * + ************************/ + + class zarray_impl_register + { + public: + + template + void insert() + { + size_t& idx = ztyped_array::get_class_static_index(); + if (idx == SIZE_MAX) + { + m_register.resize(++m_next_index); + idx = m_register.size() - 1u; + + } + else if (m_register.size() <= idx) + { + m_register.resize(idx + 1u); + } + m_register[idx] = std::unique_ptr(detail::build_zarray(std::move(xarray()))); + } + + const zarray_impl& operator[](size_t index) const + { + return *(m_register[index]); + } + + static zarray_impl_register& instance() + { + static zarray_impl_register r; + return r; + } + + private: + + zarray_impl_register() + : m_next_index(0) + { + insert(); + insert(); + } + + size_t m_next_index; + std::vector> m_register; + }; + + } #endif diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp index 82997e376..419902bf6 100644 --- a/include/xtensor/zfunction.hpp +++ b/include/xtensor/zfunction.hpp @@ -29,10 +29,14 @@ namespace xt template , self_type>::value>> zfunction(Func&& f, CTA&&... e) noexcept; + std::size_t get_result_type_index() const; + std::unique_ptr allocate_result() const; zarray_impl& assign_to(zarray_impl& res) const; private: + using dispatcher_type = zdispatcher_t; + template std::enable_if_t::value, const zarray_impl&> get_array_impl(const E& e, zarray_impl& res) const @@ -47,12 +51,46 @@ namespace xt return e.get_implementation(); } + template + std::size_t get_result_type_index_impl(std::index_sequence) const; + template zarray_impl& assign_to_impl(std::index_sequence, zarray_impl& res) const; tuple_type m_e; }; + class zarray; + + namespace detail + { + + template + struct zresult_type + { + static size_t get_index(const E& e) + { + return e.get_result_type_index(); + } + }; + + template <> + struct zresult_type + { + template + static size_t get_index(const E& e) + { + return e.get_implementation().get_class_index(); + } + }; + + template + inline size_t get_result_type_index(const E& e) + { + return zresult_type::get_index(e); + } + } + template template inline zfunction::zfunction(Func&&, CTA&&... e) noexcept @@ -60,18 +98,38 @@ namespace xt { } + template + std::size_t zfunction::get_result_type_index() const + { + return get_result_type_index_impl(std::make_index_sequence()); + } + + template + std::unique_ptr zfunction::allocate_result() const + { + std::size_t idx = get_result_type_index(); + return std::unique_ptr(zarray_impl_register::instance()[idx].clone()); + } + template inline zarray_impl& zfunction::assign_to(zarray_impl& res) const { return assign_to_impl(std::make_index_sequence(), res); } + template + template + std::size_t zfunction::get_result_type_index_impl(std::index_sequence) const + { + auto& reg = zarray_impl_register::instance(); + return dispatcher_type::get_type_index(reg[detail::get_result_type_index(std::get(m_e))]...); + } + template template inline zarray_impl& zfunction::assign_to_impl(std::index_sequence, zarray_impl& res) const { - // To do: call assign_to on zfunciton arguments - zdispatcher_t::dispatch(get_array_impl(std::get(m_e), res)..., res); + dispatcher_type::dispatch(get_array_impl(std::get(m_e), res)..., res); return res; } } diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index 4a77b51a2..ca519b271 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -69,6 +69,9 @@ namespace xt [](const double& lhs, const double& rhs) { return lhs + std::exp(rhs); }); EXPECT_TRUE(all(isclose(res, expected))); + + size_t res_index = f.get_result_type_index(); + EXPECT_EQ(res_index, ztyped_array::get_class_static_index()); } } #endif From dc5b09736c395ac8827f1aaba73618dbc0ebac43 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 3 Sep 2020 15:59:11 +0200 Subject: [PATCH 086/606] zarray_expression_tag and specialization of select_xfunction_expression --- include/xtensor/zarray.hpp | 3 ++- include/xtensor/zarray_impl.hpp | 27 +++++++++++++++++++++++++++ include/xtensor/zfunction.hpp | 13 ++++++++++++- test/test_zarray.cpp | 28 +++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 364fe2afa..94b3cd82e 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -24,10 +24,11 @@ namespace xt * zarray * **********/ - class zarray + class zarray : public xexpression { public: + using expression_tag = zarray_expression_tag; using implementation_ptr = std::unique_ptr; zarray() = default; diff --git a/include/xtensor/zarray_impl.hpp b/include/xtensor/zarray_impl.hpp index d2bd20d74..d845f8e0b 100644 --- a/include/xtensor/zarray_impl.hpp +++ b/include/xtensor/zarray_impl.hpp @@ -15,6 +15,33 @@ namespace xt { + /************************* + * zarray_expression_tag * + *************************/ + + struct zarray_expression_tag {}; + + namespace extension + { + template <> + struct expression_tag_and + { + using type = zarray_expression_tag; + }; + + template <> + struct expression_tag_and + : expression_tag_and + { + }; + + template <> + struct expression_tag_and + { + using type = zarray_expression_tag; + }; + } + /*************** * zarray_impl * ***************/ diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp index 419902bf6..d569953c8 100644 --- a/include/xtensor/zfunction.hpp +++ b/include/xtensor/zfunction.hpp @@ -18,10 +18,12 @@ namespace xt { template - class zfunction + class zfunction : public xexpression> { public: + using expression_tag = zarray_expression_tag; + using self_type = zfunction; using tuple_type = std::tuple; using functor_type = F; @@ -60,6 +62,15 @@ namespace xt tuple_type m_e; }; + namespace detail + { + template + struct select_xfunction_expression + { + using type = zfunction; + }; + } + class zarray; namespace detail diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index ca519b271..c94c81666 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -42,7 +42,7 @@ namespace xt } // TODO: move to dedicated test file - TEST(zarray, add) + TEST(zarray, zfunction) { using exp_dispatcher_type = zdispatcher_t; exp_dispatcher_type::init(); @@ -73,6 +73,32 @@ namespace xt size_t res_index = f.get_result_type_index(); EXPECT_EQ(res_index, ztyped_array::get_class_static_index()); } + + TEST(zarray, operations) + { + using exp_dispatcher_type = zdispatcher_t; + exp_dispatcher_type::init(); + + using add_dispatcher_type = zdispatcher_t; + add_dispatcher_type::init(); + + xarray a = {{0.5, 1.5}, {2.5, 3.5}}; + xarray b = {{-0.2, 2.4}, {1.3, 4.7}}; + xarray res; + + zarray za(a); + zarray zb(b); + zarray zres(res); + + auto f = za + xt::exp(zb); + f.assign_to(zres.get_implementation()); + + auto expected = xarray::from_shape({2, 2}); + std::transform(a.cbegin(), a.cend(), b.cbegin(), expected.begin(), + [](const double& lhs, const double& rhs) { return lhs + std::exp(rhs); }); + + EXPECT_TRUE(all(isclose(res, expected))); + } } #endif From 474b5d2a14f980d909e947c853cc7f18dc4f41fe Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 3 Sep 2020 16:38:57 +0200 Subject: [PATCH 087/606] Assign mechanism for zarray --- include/xtensor/zarray.hpp | 49 +++++++++++++++++++++++++++++++++++-- include/xtensor/zassign.hpp | 35 ++++++++++++++++++++++++++ test/test_zarray.cpp | 23 +++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 include/xtensor/zassign.hpp diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 94b3cd82e..8e299bc7b 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -16,6 +16,7 @@ #include "xarray.hpp" #include "zarray_impl.hpp" +#include "zassign.hpp" namespace xt { @@ -24,11 +25,20 @@ namespace xt * zarray * **********/ - class zarray : public xexpression + class zarray; + + template <> + struct xcontainer_inner_types + { + using temporary_type = zarray; + }; + + class zarray : public xcontainer_semantic { public: using expression_tag = zarray_expression_tag; + using semantic_base = xcontainer_semantic; using implementation_ptr = std::unique_ptr; zarray() = default; @@ -38,6 +48,7 @@ namespace xt zarray(E&& e); zarray(implementation_ptr&& impl); + zarray& operator=(implementation_ptr&& impl); zarray(const zarray& rhs); zarray& operator=(const zarray& rhs); @@ -45,6 +56,9 @@ namespace xt zarray(zarray&& rhs); zarray& operator=(zarray&& rhs); + template + zarray& operator=(const xexpression&); + void swap(zarray& rhs); zarray_impl& get_implementation(); @@ -58,6 +72,12 @@ namespace xt private: + template + void init_implementation(E&& e, xtensor_expression_tag); + + template + void init_implementation(const xexpression& e, zarray_expression_tag); + implementation_ptr p_impl; }; @@ -65,10 +85,23 @@ namespace xt * zarray implementation * *************************/ + template + inline void zarray::init_implementation(E&& e, xtensor_expression_tag) + { + p_impl = implementation_ptr(detail::build_zarray(std::forward(e))); + } + + template + inline void zarray::init_implementation(const xexpression& e, zarray_expression_tag) + { + p_impl = nullptr; + semantic_base::assign(e); + } + template inline zarray::zarray(E&& e) - : p_impl(detail::build_zarray(std::forward(e))) { + init_implementation(std::forward(e), extension::get_expression_tag_t>()); } inline zarray::zarray(implementation_ptr&& impl) @@ -76,6 +109,12 @@ namespace xt { } + inline zarray& zarray::operator=(implementation_ptr&& impl) + { + p_impl = std::move(impl); + return *this; + } + inline zarray::zarray(const zarray& rhs) : p_impl(rhs.p_impl->clone()) { @@ -99,6 +138,12 @@ namespace xt return *this; } + template + inline zarray& zarray::operator=(const xexpression& e) + { + return semantic_base::operator=(e); + } + inline void zarray::swap(zarray& rhs) { std::swap(p_impl, rhs.p_impl); diff --git a/include/xtensor/zassign.hpp b/include/xtensor/zassign.hpp new file mode 100644 index 000000000..36ca622a4 --- /dev/null +++ b/include/xtensor/zassign.hpp @@ -0,0 +1,35 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XTENSOR_ZASSIGN_HPP +#define XTENSOR_ZASSIGN_HPP + +#include "xassign.hpp" +#include "zarray_impl.hpp" + +namespace xt +{ + template <> + class xexpression_assigner + { + public: + + template + static void assign_xexpression(xexpression& e1, const xexpression& e2) + { + std::unique_ptr res_impl = e2.derived_cast().allocate_result(); + e2.derived_cast().assign_to(*res_impl); + e1.derived_cast() = std::move(res_impl); + } + }; + +} + +#endif + diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index c94c81666..ab6f70e82 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -99,6 +99,29 @@ namespace xt EXPECT_TRUE(all(isclose(res, expected))); } + + TEST(zarray, assign) + { + using exp_dispatcher_type = zdispatcher_t; + exp_dispatcher_type::init(); + + using add_dispatcher_type = zdispatcher_t; + add_dispatcher_type::init(); + + xarray a = {{0.5, 1.5}, {2.5, 3.5}}; + xarray b = {{-0.2, 2.4}, {1.3, 4.7}}; + + zarray za(a); + zarray zb(b); + + zarray zres = za + xt::exp(zb); + auto expected = xarray::from_shape({2, 2}); + std::transform(a.cbegin(), a.cend(), b.cbegin(), expected.begin(), + [](const double& lhs, const double& rhs) { return lhs + std::exp(rhs); }); + + const auto& res = zres.get_array(); + EXPECT_TRUE(all(isclose(res, expected))); + } } #endif From 617885e931a4f848d6d6f9522338060268644816 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Wed, 2 Sep 2020 11:09:26 +0200 Subject: [PATCH 088/606] Add xindex_path to transform indexes into path --- include/xtensor/xchunk_store_manager.hpp | 175 ++++++++++++++--------- 1 file changed, 107 insertions(+), 68 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 2fba2c001..9de768e65 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -11,38 +11,53 @@ namespace xt { + /*************************** + * xindex_path declaration * + ***************************/ + + class xindex_path + { + public: + void set_directory(const char* directory); + template + void index_to_path(I, I, std::string&); + + private: + std::string m_directory; + }; + /************************************ * xchunk_store_manager declaration * ************************************/ - template + template class xchunk_store_manager; - template - struct xcontainer_inner_types> + template + struct xcontainer_inner_types> { using storage_type = EC; using reference = EC&; using const_reference = const EC&; using size_type = std::size_t; - using temporary_type = xchunk_store_manager; + using temporary_type = xchunk_store_manager; }; - template - struct xiterable_inner_types> + template + struct xiterable_inner_types> { using inner_shape_type = std::vector; - using stepper = xindexed_stepper, false>; - using const_stepper = xindexed_stepper, true>; + using stepper = xindexed_stepper, false>; + using const_stepper = xindexed_stepper, true>; }; - template - class xchunk_store_manager: public xaccessible>, - public xiterable> + template + class xchunk_store_manager: public xaccessible>, + public xiterable> { public: - using self_type = xchunk_store_manager; + using self_type = xchunk_store_manager; using inner_types = xcontainer_inner_types; using storage_type = typename inner_types::storage_type; using value_type = storage_type; @@ -95,6 +110,7 @@ namespace xt void set_pool_size(std::size_t n); void set_directory(const char* directory); + IP& get_index_path(); void flush(); template @@ -115,15 +131,44 @@ namespace xt chunk_pool_type m_chunk_pool; index_pool_type m_index_pool; std::size_t m_unload_index; - std::string m_directory; + IP m_index_path; }; + /****************************** + * xindex_path implementation * + ******************************/ + + void xindex_path::set_directory(const char* directory) + { + m_directory = directory; + if (m_directory.back() != '/') + { + m_directory.push_back('/'); + } + } + + template + void xindex_path::index_to_path(I first, I last, std::string& path) + { + std::string fname; + for (auto it = first; it != last; ++it) + { + if (!fname.empty()) + { + fname.push_back('.'); + } + fname.append(std::to_string(*it)); + } + path = m_directory + fname; + std::cout << path << std::endl; + } + /*************************************** * xchunk_store_manager implementation * ***************************************/ - template - inline xchunk_store_manager::xchunk_store_manager() + template + inline xchunk_store_manager::xchunk_store_manager() : m_shape() // default pool size is 1 // so that first chunk is always resized to the chunk shape @@ -134,84 +179,84 @@ namespace xt m_chunk_pool[0].ignore_empty_path(true); } - template - inline auto xchunk_store_manager::shape() const noexcept -> const shape_type& + template + inline auto xchunk_store_manager::shape() const noexcept -> const shape_type& { return m_shape; } - template + template template - inline auto xchunk_store_manager::operator()(Idxs... idxs) -> reference + inline auto xchunk_store_manager::operator()(Idxs... idxs) -> reference { auto index = get_indexes(idxs...); return map_file_array(index.cbegin(), index.cend()); } - template + template template - inline auto xchunk_store_manager::operator()(Idxs... idxs) const -> const_reference + inline auto xchunk_store_manager::operator()(Idxs... idxs) const -> const_reference { auto index = get_indexes(idxs...); return map_file_array(index.cbegin(), index.cend()); } - template + template template - inline auto xchunk_store_manager::element(It first, It last) -> reference + inline auto xchunk_store_manager::element(It first, It last) -> reference { return map_file_array(first, last); } - template + template template - inline auto xchunk_store_manager::element(It first, It last) const -> const_reference + inline auto xchunk_store_manager::element(It first, It last) const -> const_reference { return map_file_array(first, last); } - template + template template - inline auto xchunk_store_manager::stepper_begin(const O& shape) noexcept -> stepper + inline auto xchunk_store_manager::stepper_begin(const O& shape) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); return stepper(this, offset); } - template + template template - inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) noexcept -> stepper + inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) noexcept -> stepper { size_type offset = shape.size() - this->dimension(); return stepper(this, offset, true); } - template + template template - inline auto xchunk_store_manager::stepper_begin(const O& shape) const noexcept -> const_stepper + inline auto xchunk_store_manager::stepper_begin(const O& shape) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); return const_stepper(this, offset); } - template + template template - inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper + inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper { size_type offset = shape.size() - this->dimension(); return const_stepper(this, offset, true); } - template + template template - inline void xchunk_store_manager::resize(S&&) + inline void xchunk_store_manager::resize(S&&) { // don't resize according to total number of chunks // instead the pool manages a number of in-memory chunks } - template - inline void xchunk_store_manager::set_pool_size(std::size_t n) + template + inline void xchunk_store_manager::set_pool_size(std::size_t n) { // first chunk always has the correct shape // get the shape before resizing the pool @@ -227,8 +272,8 @@ namespace xt } } - template - inline void xchunk_store_manager::flush() + template + inline void xchunk_store_manager::flush() { for (auto& chunk: m_chunk_pool) { @@ -236,9 +281,9 @@ namespace xt } } - template + template template - void xchunk_store_manager::configure_format(C& config) + void xchunk_store_manager::configure_format(C& config) { for (auto& chunk: m_chunk_pool) { @@ -246,41 +291,33 @@ namespace xt } } - template - void xchunk_store_manager::set_directory(const char* directory) + template + IP& xchunk_store_manager::get_index_path() { - m_directory = directory; - if (m_directory.back() != '/') - { - m_directory.append("/"); - } + return m_index_path; } - template + template + void xchunk_store_manager::set_directory(const char* directory) + { + m_index_path.set_directory(directory); + } + + template template - inline auto xchunk_store_manager::map_file_array(I first, I last) -> reference + inline auto xchunk_store_manager::map_file_array(I first, I last) -> reference { std::string path; - std::string fname; - std::vector index; - for (auto it = first; it != last; ++it) - { - if (!fname.empty()) - { - fname.append("."); - } - fname.append(std::to_string(*it)); - index.push_back(*it); - } - path = m_directory + fname; - if (index.empty()) + m_index_path.index_to_path(first, last, path); + if (first == last) { return m_chunk_pool[0]; } else { // check if the chunk is already loaded in memory - const auto it1 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), index); + const auto it1 = std::find_if(m_index_pool.cbegin(), m_index_pool.cend(), [first, last](const auto& v) + { return std::equal(v.cbegin(), v.cend(), first, last); }); std::size_t i; if (it1 != m_index_pool.cend()) { @@ -294,23 +331,25 @@ namespace xt { i = std::distance(m_index_pool.cbegin(), it2); m_chunk_pool[i].set_path(path); - m_index_pool[i] = index; + m_index_pool[i].resize(static_cast(std::distance(first, last))); + std::copy(first, last, m_index_pool[i].begin()); return m_chunk_pool[i]; } // no free chunk, take one (which will thus be unloaded) // fairness is guaranteed through the use of a walking index m_chunk_pool[m_unload_index].set_path(path); - m_index_pool[m_unload_index] = index; + m_index_pool[m_unload_index].resize(static_cast(std::distance(first, last))); + std::copy(first, last, m_index_pool[m_unload_index].begin()); auto& chunk = m_chunk_pool[m_unload_index]; m_unload_index = (m_unload_index + 1) % m_index_pool.size(); return chunk; } } - template + template template inline std::array - xchunk_store_manager::get_indexes(Idxs... idxs) const + xchunk_store_manager::get_indexes(Idxs... idxs) const { std::array indexes = {{idxs...}}; return indexes; From 0f911a585f097fb10fb1188d8ef751b8989eaa4f Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 3 Sep 2020 23:04:03 +0200 Subject: [PATCH 089/606] zfunction refactoring --- include/xtensor/zfunction.hpp | 59 ++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp index d569953c8..4af97b543 100644 --- a/include/xtensor/zfunction.hpp +++ b/include/xtensor/zfunction.hpp @@ -31,28 +31,14 @@ namespace xt template , self_type>::value>> zfunction(Func&& f, CTA&&... e) noexcept; - std::size_t get_result_type_index() const; std::unique_ptr allocate_result() const; + std::size_t get_result_type_index() const; zarray_impl& assign_to(zarray_impl& res) const; private: using dispatcher_type = zdispatcher_t; - template - std::enable_if_t::value, const zarray_impl&> - get_array_impl(const E& e, zarray_impl& res) const - { - return e.assign_to(res); - } - - template - std::enable_if_t::value, const zarray_impl&> - get_array_impl(E& e, zarray_impl&) const - { - return e.get_implementation(); - } - template std::size_t get_result_type_index_impl(std::index_sequence) const; @@ -71,34 +57,55 @@ namespace xt }; } + /**************************** + * zfunction implementation * + ****************************/ + class zarray; namespace detail { template - struct zresult_type + struct zfunction_argument { - static size_t get_index(const E& e) + static std::size_t get_index(const E& e) { return e.get_result_type_index(); } + + static const zarray_impl& get_array_impl(const E& e, zarray_impl& res) + { + return e.assign_to(res); + } }; template <> - struct zresult_type + struct zfunction_argument { template - static size_t get_index(const E& e) + static std::size_t get_index(const E& e) { return e.get_implementation().get_class_index(); } + + template + static const zarray_impl& get_array_impl(const E& e, zarray_impl&) + { + return e.get_implementation(); + } }; template inline size_t get_result_type_index(const E& e) { - return zresult_type::get_index(e); + return zfunction_argument::get_index(e); + } + + template + inline const zarray_impl& get_array_impl(const E& e, zarray_impl& z) + { + return zfunction_argument::get_array_impl(e, z); } } @@ -110,16 +117,16 @@ namespace xt } template - std::size_t zfunction::get_result_type_index() const + std::unique_ptr zfunction::allocate_result() const { - return get_result_type_index_impl(std::make_index_sequence()); + std::size_t idx = get_result_type_index(); + return std::unique_ptr(zarray_impl_register::instance()[idx].clone()); } template - std::unique_ptr zfunction::allocate_result() const + std::size_t zfunction::get_result_type_index() const { - std::size_t idx = get_result_type_index(); - return std::unique_ptr(zarray_impl_register::instance()[idx].clone()); + return get_result_type_index_impl(std::make_index_sequence()); } template @@ -140,7 +147,7 @@ namespace xt template inline zarray_impl& zfunction::assign_to_impl(std::index_sequence, zarray_impl& res) const { - dispatcher_type::dispatch(get_array_impl(std::get(m_e), res)..., res); + dispatcher_type::dispatch(detail::get_array_impl(std::get(m_e), res)..., res); return res; } } From b2a6b185bbbf771e768f12f2c71ee092ef1bf8bd Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 4 Sep 2020 09:32:17 +0200 Subject: [PATCH 090/606] zdispatcher refactoring --- include/xtensor/zdispatcher.hpp | 307 +++++++++++++++++++++----------- include/xtensor/zfunction.hpp | 9 +- 2 files changed, 207 insertions(+), 109 deletions(-) diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp index 3fd3f5fed..8994df631 100644 --- a/include/xtensor/zdispatcher.hpp +++ b/include/xtensor/zdispatcher.hpp @@ -49,50 +49,31 @@ namespace xt template class zdouble_dispatcher { - private: + public: - using zfunctor_type = get_zmapped_functor_t; - using ztype_dispatcher = ztype_dispatcher_impl>; - using zrun_dispatcher = zrun_dispatcher_impl>; - - static ztype_dispatcher& type_dispatcher() - { - static ztype_dispatcher dispatcher; - return dispatcher; - } + template + static void insert(); - static zrun_dispatcher& run_dispatcher() - { - static zrun_dispatcher dispatcher; - return dispatcher; - } + static void init(); + static void dispatch(const zarray_impl& z1, zarray_impl& res); + static size_t get_type_index(const zarray_impl& z1); - public: + private: - template - static void insert() - { - using arg_type = const ztyped_array; - using res_type = ztyped_array; - run_dispatcher().template insert(&zfunctor_type::template run); - type_dispatcher().template insert(&zfunctor_type::template index); - } + static zdouble_dispatcher& instance(); - static void init() - { - insert(); - insert(); - } + zdouble_dispatcher(); + ~zdouble_dispatcher() = default; - static void dispatch(const zarray_impl& z1, zarray_impl& res) - { - run_dispatcher().dispatch(z1, res); - } + template + void insert_impl(); - static size_t get_type_index(const zarray_impl& z1) - { - return type_dispatcher().dispatch(z1); - } + using zfunctor_type = get_zmapped_functor_t; + using ztype_dispatcher = ztype_dispatcher_impl>; + using zrun_dispatcher = zrun_dispatcher_impl>; + + ztype_dispatcher m_type_dispatcher; + zrun_dispatcher m_run_dispatcher; }; /********************** @@ -105,51 +86,31 @@ namespace xt template class ztriple_dispatcher { - private: + public: - using zfunctor_type = get_zmapped_functor_t; - using ztype_dispatcher = ztype_dispatcher_impl>; - using zrun_dispatcher = zrun_dispatcher_impl>; - - static ztype_dispatcher& type_dispatcher() - { - static ztype_dispatcher dispatcher; - return dispatcher; - } + template + static void insert(); - static zrun_dispatcher& run_dispatcher() - { - static zrun_dispatcher dispatcher; - return dispatcher; - } + static void init(); + static void dispatch(const zarray_impl& z1, const zarray_impl& z2, zarray_impl& res); + static size_t get_type_index(const zarray_impl& z1, const zarray_impl& z2); - public: + private: - template - static void insert() - { - using arg_type1 = const ztyped_array; - using arg_type2 = const ztyped_array; - using res_type = ztyped_array; - run_dispatcher().template insert(&zfunctor_type::template run); - type_dispatcher().template insert(&zfunctor_type::template index); - } + static ztriple_dispatcher& instance(); - static void init() - { - insert(); - insert(); - } + ztriple_dispatcher(); + ~ztriple_dispatcher() = default; - static void dispatch(const zarray_impl& z1, const zarray_impl& z2, zarray_impl& res) - { - run_dispatcher().dispatch(z1, z2, res); - } + template + void insert_impl(); - static size_t get_type_index(const zarray_impl& z1, const zarray_impl& z2) - { - return type_dispatcher().dispatch(z1, z2); - } + using zfunctor_type = get_zmapped_functor_t; + using ztype_dispatcher = ztype_dispatcher_impl>; + using zrun_dispatcher = zrun_dispatcher_impl>; + + ztype_dispatcher m_type_dispatcher; + zrun_dispatcher m_run_dispatcher; }; /*************** @@ -183,47 +144,181 @@ namespace xt public: template - void insert() - { - size_t& idx = ztyped_array::get_class_static_index(); - if (idx == SIZE_MAX) - { - m_register.resize(++m_next_index); - idx = m_register.size() - 1u; - - } - else if (m_register.size() <= idx) - { - m_register.resize(idx + 1u); - } - m_register[idx] = std::unique_ptr(detail::build_zarray(std::move(xarray()))); - } + static void insert(); - const zarray_impl& operator[](size_t index) const - { - return *(m_register[index]); - } - - static zarray_impl_register& instance() - { - static zarray_impl_register r; - return r; - } + static void init(); + static const zarray_impl& get(size_t index); private: - zarray_impl_register() - : m_next_index(0) - { - insert(); - insert(); - } + static zarray_impl_register& instance(); + + zarray_impl_register(); + ~zarray_impl_register() = default; + + template + void insert_impl(); size_t m_next_index; std::vector> m_register; }; + /************************************* + * zdouble_dispatcher implementation * + *************************************/ + + template + template + inline void zdouble_dispatcher::insert() + { + instance().template insert_impl(); + } + + template + inline void zdouble_dispatcher::init() + { + instance(); + } + + template + inline void zdouble_dispatcher::dispatch(const zarray_impl& z1, zarray_impl& res) + { + instance().m_run_dispatcher.dispatch(z1, res); + } + + template + inline size_t zdouble_dispatcher::get_type_index(const zarray_impl& z1) + { + return instance().m_type_dispatcher.dispatch(z1); + } + + template + inline zdouble_dispatcher& zdouble_dispatcher::instance() + { + static zdouble_dispatcher inst; + return inst; + } + + template + inline zdouble_dispatcher::zdouble_dispatcher() + { + insert_impl(); + insert_impl(); + } + template + template + inline void zdouble_dispatcher::insert_impl() + { + using arg_type = const ztyped_array; + using res_type = ztyped_array; + m_run_dispatcher.template insert(&zfunctor_type::template run); + m_type_dispatcher.template insert(&zfunctor_type::template index); + } + + /************************************* + * ztriple_dispatcher implementation * + *************************************/ + + template + template + inline void ztriple_dispatcher::insert() + { + instance().template insert_impl(); + } + + template + inline void ztriple_dispatcher::init() + { + instance(); + } + + template + inline void ztriple_dispatcher::dispatch(const zarray_impl& z1, const zarray_impl& z2, zarray_impl& res) + { + instance().m_run_dispatcher.dispatch(z1, z2, res); + } + + template + inline size_t ztriple_dispatcher::get_type_index(const zarray_impl& z1, const zarray_impl& z2) + { + return instance().m_type_dispatcher.dispatch(z1, z2); + } + + template + inline ztriple_dispatcher& ztriple_dispatcher::instance() + { + static ztriple_dispatcher inst; + return inst; + } + + template + inline ztriple_dispatcher::ztriple_dispatcher() + { + insert_impl(); + insert_impl(); + } + + template + template + inline void ztriple_dispatcher::insert_impl() + { + using arg_type1 = const ztyped_array; + using arg_type2 = const ztyped_array; + using res_type = ztyped_array; + m_run_dispatcher.template insert(&zfunctor_type::template run); + m_type_dispatcher.template insert(&zfunctor_type::template index); + } + + /*************************************** + * zarray_impl_register implementation * + ***************************************/ + + template + inline void zarray_impl_register::insert() + { + instance().template insert_impl(); + } + + inline void zarray_impl_register::init() + { + instance(); + } + + inline const zarray_impl& zarray_impl_register::get(size_t index) + { + return *(instance().m_register[index]); + } + + inline zarray_impl_register& zarray_impl_register::instance() + { + static zarray_impl_register r; + return r; + } + + inline zarray_impl_register::zarray_impl_register() + : m_next_index(0) + { + insert_impl(); + insert_impl(); + } + + template + inline void zarray_impl_register::insert_impl() + { + size_t& idx = ztyped_array::get_class_static_index(); + if (idx == SIZE_MAX) + { + m_register.resize(++m_next_index); + idx = m_register.size() - 1u; + + } + else if (m_register.size() <= idx) + { + m_register.resize(idx + 1u); + } + m_register[idx] = std::unique_ptr(detail::build_zarray(std::move(xarray()))); + } } #endif diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp index 4af97b543..0f6fb55ea 100644 --- a/include/xtensor/zfunction.hpp +++ b/include/xtensor/zfunction.hpp @@ -120,7 +120,7 @@ namespace xt std::unique_ptr zfunction::allocate_result() const { std::size_t idx = get_result_type_index(); - return std::unique_ptr(zarray_impl_register::instance()[idx].clone()); + return std::unique_ptr(zarray_impl_register::get(idx).clone()); } template @@ -139,8 +139,11 @@ namespace xt template std::size_t zfunction::get_result_type_index_impl(std::index_sequence) const { - auto& reg = zarray_impl_register::instance(); - return dispatcher_type::get_type_index(reg[detail::get_result_type_index(std::get(m_e))]...); + return dispatcher_type::get_type_index( + zarray_impl_register::get( + detail::get_result_type_index(std::get(m_e)) + )... + ); } template From cba98446c3dfdc35323e3f03d2e130a2e77b5907 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 4 Sep 2020 11:57:00 +0200 Subject: [PATCH 091/606] Populated zfunctors --- include/xtensor/zmath.hpp | 200 +++++++++++++++++++++++++++++--------- test/test_zarray.cpp | 2 +- 2 files changed, 153 insertions(+), 49 deletions(-) diff --git a/include/xtensor/zmath.hpp b/include/xtensor/zmath.hpp index 0d1d0e758..5d4c7b92a 100644 --- a/include/xtensor/zmath.hpp +++ b/include/xtensor/zmath.hpp @@ -33,60 +33,164 @@ namespace xt template using get_zmapped_functor_t = typename get_zmapped_functor::type; -/*#define DEFINE_ZFUNCTOR_MAPPING(XF, ZF) \ - template <> \ - struct get_zmapped_functor \ - { using type = ZF; } -*/ - struct zadd - { - template - static void run(const ztyped_array& z1, - const ztyped_array& z2, - ztyped_array& zres) - { - detail::zassign_data(zres.get_array(), z1.get_array() + z2.get_array()); - } +#define XTENSOR_ZMAPPED_FUNCTOR(ZFUN, XFUN) \ + template <> \ + struct get_zmapped_functor \ + { using type = ZFUN; } - template - static size_t index(const ztyped_array&, const ztyped_array&) - { - using result_type = ztyped_array() + std::declval())>; - return result_type::get_class_static_index(); - } - }; +#define XTENSOR_UNARY_ZOPERATOR(ZNAME, XOP, XFUN) \ + struct ZNAME \ + { \ + template \ + static void run(const ztyped_array& z, const ztyped_array& zres) \ + { \ + detail::zassign_data(zres.get_array(), XOP z.get_array()); \ + } \ + template \ + static size_t index(const ztyped_array&) \ + { \ + using result_type = ztyped_array())>; \ + return result_type::get_class_static_index(); \ + } \ + }; \ + XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - template <> - struct get_zmapped_functor - { - using type = zadd; - }; +#define ZCONCAT(X, Y) X Y +#define XTENSOR_BINARY_ZOPERATOR(ZNAME, XOP, XFUN) \ + struct ZNAME \ + { \ + template \ + static void run(const ztyped_array& z1, \ + const ztyped_array& z2, \ + ztyped_array& zres) \ + { \ + detail::zassign_data(zres.get_array(), \ + z1.get_array() XOP z2.get_array()); \ + } \ + template \ + static size_t index(const ztyped_array&, const ztyped_array&) \ + { \ + using result_type = \ + ztyped_array() XOP std::declval())>; \ + return result_type::get_class_static_index(); \ + } \ + }; \ + XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - //DEFINE_ZFUNCTOR_MAPPING((detail::plus), zadd); +#define XTENSOR_UNARY_ZFUNCTOR(ZNAME, XEXP, XFUN) \ + struct ZNAME \ + { \ + template \ + static void run(const ztyped_array& z, \ + ztyped_array& zres) \ + { \ + detail::zassign_data(zres.get_array(), XEXP(z.get_array())); \ + } \ + template \ + static size_t index(const ztyped_array&) \ + { \ + using value_type = decltype(std::declval()(std::declval())); \ + return ztyped_array::get_class_static_index(); \ + } \ + }; \ + XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - struct zexp - { - template - static void run(const ztyped_array& z, - ztyped_array& zres) - { - detail::zassign_data(zres.get_array(), xt::exp(z.get_array())); - } +#define XTENSOR_BINARY_ZFUNCTOR(ZNAME, XEXP, XFUN) \ + struct ZNAME \ + { \ + template \ + static void run(const ztyped_array& z1, \ + const ztyped_array& z2, \ + ztyped_array& zres) \ + { \ + detail::zassign_data(zres.get_array(), \ + XEXP(z1.get_array(), z2.get_array())); \ + } \ + template \ + static size_t index(const ztyped_array&, const ztyped_array&) \ + { \ + using value_type = decltype( \ + std::declval()(std::declval(), std::declval())); \ + return ztyped_array::get_class_static_index(); \ + } \ + }; \ + XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - template - static size_t index(const ztyped_array&) - { - using value_type = decltype(std::declval()(std::declval())); - return ztyped_array::get_class_static_index(); - } - }; + XTENSOR_UNARY_ZOPERATOR(zidentity, +, detail::identity); + XTENSOR_UNARY_ZOPERATOR(znegate, -, detail::negate); + XTENSOR_BINARY_ZOPERATOR(zplus, +, detail::plus); + XTENSOR_BINARY_ZOPERATOR(zminus, -, detail::minus); + XTENSOR_BINARY_ZOPERATOR(zmultiuplies, *, detail::multiplies); + XTENSOR_BINARY_ZOPERATOR(zdivides, /, detail::divides); + XTENSOR_BINARY_ZOPERATOR(zmodulus, %, detail::modulus); + XTENSOR_BINARY_ZOPERATOR(zlogical_or, ||, detail::logical_or); + XTENSOR_BINARY_ZOPERATOR(zlogical_and, &&, detail::logical_and); + XTENSOR_UNARY_ZOPERATOR(zlogical_not, !, detail::logical_not); + XTENSOR_BINARY_ZOPERATOR(zbitwise_or, |, detail::bitwise_or); + XTENSOR_BINARY_ZOPERATOR(zbitwise_and, &, detail::bitwise_and); + XTENSOR_BINARY_ZOPERATOR(zbitwise_xor, ^, detail::bitwise_xor); + XTENSOR_UNARY_ZOPERATOR(zbitwise_not, ~, detail::bitwise_not); + XTENSOR_BINARY_ZOPERATOR(zleft_shift, <<, detail::left_shift); + XTENSOR_BINARY_ZOPERATOR(zright_shift, >>, detail::right_shift); + XTENSOR_BINARY_ZOPERATOR(zless, <, detail::less); + XTENSOR_BINARY_ZOPERATOR(zless_equal, <=, detail::less_equal); + XTENSOR_BINARY_ZOPERATOR(zgreater, >, detail::greater); + XTENSOR_BINARY_ZOPERATOR(zgreater_equal, >=, detail::greater_equal); + XTENSOR_BINARY_ZOPERATOR(zequal_to, ==, detail::equal_to); + XTENSOR_BINARY_ZOPERATOR(znot_equal_to, !=, detail::not_equal_to); + + + XTENSOR_UNARY_ZFUNCTOR(zfabs, xt::fabs, math::fabs_fun); + XTENSOR_BINARY_ZFUNCTOR(zfmod, xt::fmod, math::fmod_fun); + XTENSOR_BINARY_ZFUNCTOR(zremainder, xt::remainder, math::remainder_fun); + //XTENSOR_TERNARY_ZFUNCTOR(fma); + XTENSOR_BINARY_ZFUNCTOR(zfmax, xt::fmax, math::fmax_fun); + XTENSOR_BINARY_ZFUNCTOR(zfmin, xt::fmin, math::fmin_fun); + XTENSOR_BINARY_ZFUNCTOR(zfdim, xt::fdim, math::fdim_fun); + XTENSOR_UNARY_ZFUNCTOR(zexp, xt::exp, math::exp_fun); + XTENSOR_UNARY_ZFUNCTOR(zexp2, xt::exp2, math::exp2_fun); + XTENSOR_UNARY_ZFUNCTOR(zexpm1, xt::expm1, math::expm1_fun); + XTENSOR_UNARY_ZFUNCTOR(zlog, xt::log, math::log_fun); + XTENSOR_UNARY_ZFUNCTOR(zlog10, xt::log10, math::log10_fun); + XTENSOR_UNARY_ZFUNCTOR(zlog2, xt::log2, math::log2_fun); + XTENSOR_UNARY_ZFUNCTOR(zlog1p, xt::log1p, math::log1p_fun); + XTENSOR_BINARY_ZFUNCTOR(zpow, xt::pow, math::pow_fun); + XTENSOR_UNARY_ZFUNCTOR(zsqrt, xt::sqrt, math::sqrt_fun); + XTENSOR_UNARY_ZFUNCTOR(zcbrt, xt::cbrt, math::cbrt_fun); + XTENSOR_BINARY_ZFUNCTOR(zhypot, xt::hypot, math::hypot_fun); + XTENSOR_UNARY_ZFUNCTOR(zsin, xt::sin, math::sin_fun); + XTENSOR_UNARY_ZFUNCTOR(zcos, xt::cos, math::cos_fun); + XTENSOR_UNARY_ZFUNCTOR(ztan, xt::tan, math::tan_fun); + XTENSOR_UNARY_ZFUNCTOR(zasin, xt::asin, math::asin_fun); + XTENSOR_UNARY_ZFUNCTOR(zacos, xt::acos, math::acos_fun); + XTENSOR_UNARY_ZFUNCTOR(zatan, xt::atan, math::atan_fun); + XTENSOR_BINARY_ZFUNCTOR(zatan2, xt::atan2, math::atan2_fun); + XTENSOR_UNARY_ZFUNCTOR(zsinh, xt::sinh, math::sinh_fun); + XTENSOR_UNARY_ZFUNCTOR(zcosh, xt::cosh, math::cosh_fun); + XTENSOR_UNARY_ZFUNCTOR(ztanh, xt::tanh, math::tanh_fun); + XTENSOR_UNARY_ZFUNCTOR(zasinh, xt::asinh, math::asinh_fun); + XTENSOR_UNARY_ZFUNCTOR(zacosh, xt::acosh, math::acosh_fun); + XTENSOR_UNARY_ZFUNCTOR(zatanh, xt::atanh, math::atanh_fun); + XTENSOR_UNARY_ZFUNCTOR(zerf, xt::erf, math::erf_fun); + XTENSOR_UNARY_ZFUNCTOR(zerfc, xt::erfc, math::erfc_fun); + XTENSOR_UNARY_ZFUNCTOR(ztgamma, xt::tgamma, math::tgamma_fun); + XTENSOR_UNARY_ZFUNCTOR(zlgamma, xt::lgamma, math::lgamma_fun); + XTENSOR_UNARY_ZFUNCTOR(zceil, xt::ceil, math::ceil_fun); + XTENSOR_UNARY_ZFUNCTOR(zfloor, xt::floor, math::floor_fun); + XTENSOR_UNARY_ZFUNCTOR(ztrunc, xt::trunc, math::trunc_fun); + XTENSOR_UNARY_ZFUNCTOR(zround, xt::round, math::round_fun); + XTENSOR_UNARY_ZFUNCTOR(znearbyint, xt::nearbyint, math::nearbyint_fun); + XTENSOR_UNARY_ZFUNCTOR(zrint, xt::rint, math::rint_fun); + XTENSOR_UNARY_ZFUNCTOR(zisfinite, xt::isfinite, math::isfinite_fun); + XTENSOR_UNARY_ZFUNCTOR(zisinf, xt::isinf, math::isinf_fun); + XTENSOR_UNARY_ZFUNCTOR(zisnan, xt::isnan, math::isnan_fun); + +#undef XTENSOR_BINARY_ZFUNCTOR +#undef XTENSOR_UNARY_ZFUNCTOR +#undef XTENSOR_BINARY_ZOPERATOR +#undef XTENSOR_UNARY_ZOPERATOR +#undef XTENSOR_ZMAPPED_FUNCTOR - template <> - struct get_zmapped_functor - { - using type = zexp; - }; - //DEFINE_ZFUNCTOR_MAPPING((math::exp_fun), zexp); } #endif diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index ab6f70e82..9741af8d1 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -61,7 +61,7 @@ namespace xt zarray zb(b); zarray zres(res); - zfunction_type f(zadd(), za, nested_zfunction_type(zexp(), zb)); + zfunction_type f(zplus(), za, nested_zfunction_type(zexp(), zb)); f.assign_to(zres.get_implementation()); auto expected = xarray::from_shape({2, 2}); From 93bb1404b7b7ba6056eaf664b2e4dc904cdbe3a0 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 4 Sep 2020 16:26:48 +0200 Subject: [PATCH 092/606] Explicit call to dispatchers initialization --- include/xtensor/zdispatcher.hpp | 103 ++++++++++++++++++++++++++++++++ include/xtensor/zmath.hpp | 2 +- 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp index 8994df631..70cd756e7 100644 --- a/include/xtensor/zdispatcher.hpp +++ b/include/xtensor/zdispatcher.hpp @@ -163,6 +163,19 @@ namespace xt std::vector> m_register; }; + /**************** + * init_zsystem * + ****************/ + + // Early initialization of all dispatchers + // and zarray_impl_register + // return int so it can be assigned to a + // static variable and be automatically + // called when loading a shared library + // for instance. + + int init_zsystem(); + /************************************* * zdouble_dispatcher implementation * *************************************/ @@ -319,6 +332,96 @@ namespace xt } m_register[idx] = std::unique_ptr(detail::build_zarray(std::move(xarray()))); } + + /******************************* + * init_zsystem implementation * + *******************************/ + + namespace detail + { + inline void init_zdispatchers() + { + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + //zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + //zdispatcher_t::init(); + //zdispatcher_t::init(); + //zdispatcher_t::init(); + //zdispatcher_t::init(); + //zdispatcher_t::init(); + //zdispatcher_t::init(); + //zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + //zdispatcher_t::init(); + //zdispatcher_t::init(); + } + } + + namespace math + { + inline void init_zdispatchers() + { + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + zdispatcher_t::init(); + } + } + + inline int init_zsystem() + { + detail::init_zdispatchers(); + math::init_zdispatchers(); + return 0; + } } #endif diff --git a/include/xtensor/zmath.hpp b/include/xtensor/zmath.hpp index 5d4c7b92a..24f1ff1b9 100644 --- a/include/xtensor/zmath.hpp +++ b/include/xtensor/zmath.hpp @@ -42,7 +42,7 @@ namespace xt struct ZNAME \ { \ template \ - static void run(const ztyped_array& z, const ztyped_array& zres) \ + static void run(const ztyped_array& z, ztyped_array& zres) \ { \ detail::zassign_data(zres.get_array(), XOP z.get_array()); \ } \ From 6976c9634ea031fb2084027c0714df5ba5faaae2 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Tue, 8 Sep 2020 11:24:52 +0200 Subject: [PATCH 093/606] [docs] Running spell-check --- docs/source/bindings.rst | 2 +- docs/source/builder.rst | 4 ++-- docs/source/closure-semantics.rst | 4 ++-- docs/source/compilers.rst | 4 ++-- docs/source/container.rst | 6 +++--- docs/source/dev-build-options.rst | 4 ++-- docs/source/expression.rst | 2 +- docs/source/histogram.rst | 2 +- docs/source/indices.rst | 4 ++-- docs/source/view.rst | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/source/bindings.rst b/docs/source/bindings.rst index 52180d89d..6397f61a3 100644 --- a/docs/source/bindings.rst +++ b/docs/source/bindings.rst @@ -276,7 +276,7 @@ Conclusion Each solution has its pros and cons and choosing one of them should be done according to the flexibility you want to impose on your API and the constraints you are imposed by the implementation. For instance, a method that requires a -lot of typing in the bindings might not suit for libraries with a huge amount of functions to expose, while a full +lot of typing in the bindings might not suit for libraries with a huge number of functions to expose, while a full generic API might be problematic if the implementation expects containers only. Below is a summary of the advantages and drawbacks of the different options: diff --git a/docs/source/builder.rst b/docs/source/builder.rst index d728c8811..418f85087 100644 --- a/docs/source/builder.rst +++ b/docs/source/builder.rst @@ -31,8 +31,8 @@ Joining expressions - ``concatenate(tuple, axis=0)``: concatenates a list of expressions along the given axis. - ``stack(tuple, axis=0)``: stacks a list of expressions along the given axis. -- ``hstack(tuple)``: stacks expressions in seqeunce horizontally (i.e. column wise). -- ``vstack(tuple)``: stacks expressions in seqeunce vertically (i.e. row wise). +- ``hstack(tuple)``: stacks expressions in sequence horizontally (i.e. column-wise). +- ``vstack(tuple)``: stacks expressions in sequence vertically (i.e. row wise). Random distributions -------------------- diff --git a/docs/source/closure-semantics.rst b/docs/source/closure-semantics.rst index 47ecf850f..af0de8d34 100644 --- a/docs/source/closure-semantics.rst +++ b/docs/source/closure-semantics.rst @@ -32,7 +32,7 @@ The two main requirements are the following: It is important for the closure type not to be a reference when the passed argument is an rvalue, which can result in dangling references. Following the conventions of the C++ standard library for naming type traits, we provide two type traits classes providing an implementation of these rules -in the ``xutils.hpp`` header, ``closure_type``, and ``const_closure_type``. The latter adds the const qualifier to the reference even when the provided argument is not const. +in the ``xutils.hpp`` header, ``closure_type``, and ``const_closure_type``. The latter adds the ``const`` qualifier to the reference even when the provided argument is not const. .. code:: cpp @@ -174,7 +174,7 @@ upon access or assignment. - In order to perform the division, the expression must hold the values or references on the numerator and denominator. - Since ``s`` is a local variable, it will be destroyed upon leaving the scope of the function, and more importantly, it is an *lvalue*. -- A consequence of ``s`` being an lvalue and a local variable, is that the ``s / value_type(size)`` would end up holding a dangling const reference on ``s``. +- A consequence of ``s`` being an lvalue and a local variable, is that the ``s / value_type(size)`` would end up holding a dangling ``const`` reference on ``s``. - Hence we must call return ``std::move(s) / value_type(size)``. The other place in this example where the C++ move semantics is used is the line ``s = sum(std::forward(e))``. The goal is to have the unevaluated ``s`` expression diff --git a/docs/source/compilers.rst b/docs/source/compilers.rst index 1a6bf31b2..a98c514d4 100644 --- a/docs/source/compilers.rst +++ b/docs/source/compilers.rst @@ -41,7 +41,7 @@ definition. Visual Studio 2017 (15.7.1) seeing declarations as extra overloads ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In ``xvectorize.hpp``, Visual Studio 15.7.1 sees the forward declaration of ``vectorize(E&&)`` as a separate ovarload. +In ``xvectorize.hpp``, Visual Studio 15.7.1 sees the forward declaration of ``vectorize(E&&)`` as a separate overload. Visual Studio 2017 double non-class parameter pack expansion ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -82,7 +82,7 @@ In this case, we polyfill the proper standard names using the deprecated ``std::has_trivial_default_constructor``. This must also be done when the compiler is clang when it makes use of the GCC implementation of the STL, which is the default behavior on linux. Properly detecting the version of the -GCC STL used by clang cannot be done with the ``__GNUC__`` macro, which are +GCC STL used by clang cannot be done with the ``__GNUC__`` macro, which is overridden by clang. Instead, we check for the definition of the macro ``_GLIBCXX_USE_CXX11_ABI`` which is only defined with GCC versions greater than ``5``. diff --git a/docs/source/container.rst b/docs/source/container.rst index 68640679b..400f28c1c 100644 --- a/docs/source/container.rst +++ b/docs/source/container.rst @@ -14,12 +14,12 @@ A multi-dimensional array of `xtensor` consists of a contiguous one-dimensional unsigned integers to the location of an element in the buffer. The range in which the indices can vary is specified by the `shape` of the array. -The scheme used to map indices into a location in the buffer is a strided indexing scheme. In such a scheme, the index ``(i0, ..., in)`` corresponds to the offset ``sum(ik * sk)`` from the beginning of the one-dimensional buffer, where ``(s0, ..., sn)`` are the `strides` of the array. Some particular cases of strided schemes implement well known memory layouts: +The scheme used to map indices into a location in the buffer is a strided indexing scheme. In such a scheme, the index ``(i0, ..., in)`` corresponds to the offset ``sum(ik * sk)`` from the beginning of the one-dimensional buffer, where ``(s0, ..., sn)`` are the `strides` of the array. Some particular cases of strided schemes implement well-known memory layouts: - the row-major layout (or C layout) is a strided index scheme where the strides grow from right to left - the column-major layout (or Fortran layout) is a strided index scheme where the strides grow from left to right -``xtensor`` provides a ``layout_type`` enum that helps to specify the layout used by multi-dimensional arrays. This enum can be used in two ways: +``xtensor`` provides a ``layout_type`` enum that helps to specify the layout used by multidimensional arrays. This enum can be used in two ways: - at compile time, as a template argument. The value ``layout_type::dynamic`` allows specifying any strided index scheme at runtime (including row-major and column-major schemes), while ``layout_type::row_major`` and ``layout_type::column_major`` fixes the strided index scheme and disable ``resize`` and constructor overloads taking a set of strides or a layout value as parameter. The default value of the template parameter is ``XTENSOR_DEFAULT_LAYOUT``. - at runtime if the previous template parameter was set to ``layout_type::dynamic``. In that case, ``resize`` and constructor overloads allow specifying a set of strides or a layout value to avoid strides computation. If neither strides nor layout is specified when instantiating or resizing a multi-dimensional array, strides corresponding to ``XTENSOR_DEFAULT_LAYOUT`` are used. @@ -62,7 +62,7 @@ However, in the latter case, the layout of the array is forced to ``row_major`` Runtime vs Compile-time dimensionality -------------------------------------- -Three container classes implementing multi-dimensional arrays are provided: ``xarray`` and ``xtensor`` and ``xtensor_fixed``. +Three container classes implementing multidimensional arrays are provided: ``xarray`` and ``xtensor`` and ``xtensor_fixed``. - ``xarray`` can be reshaped dynamically to any number of dimensions. It is the container that is the most similar to numpy arrays. - ``xtensor`` has a dimension set at compilation time, which enables many optimizations. For example, shapes and strides diff --git a/docs/source/dev-build-options.rst b/docs/source/dev-build-options.rst index 5d3791390..e662324a8 100644 --- a/docs/source/dev-build-options.rst +++ b/docs/source/dev-build-options.rst @@ -22,7 +22,7 @@ Build on your system. - ``XTENSOR_USE_TBB``: enables parallel assignment loop. This requires that you have you have tbb_ installed on your system. -- ``XTENSOR_USE_OPENMP``: enables parallel assignment loop using OpenMP. This requires that OpenMP is avaliable on your system. +- ``XTENSOR_USE_OPENMP``: enables parallel assignment loop using OpenMP. This requires that OpenMP is available on your system. All these options are disabled by default. Enabling ``DOWNLOAD_GTEST`` or setting ``GTEST_SRC_DIR`` enables ``BUILD_TESTS``. @@ -66,7 +66,7 @@ including any of its header. Here is a list of available macros: on your system. - ``XTENSOR_USE_TBB``: enables parallel assignment loop. This requires that you have you have tbb_ installed on your system. -- ``XTENSOR_USE_OPENMP``: enables parallel assignment loop using OpenMP. This requires that OpenMP is avaliable on your system. +- ``XTENSOR_USE_OPENMP``: enables parallel assignment loop using OpenMP. This requires that OpenMP is available on your system. - ``XTENSOR_DEFAULT_DATA_CONTAINER(T, A)``: defines the type used as the default data container for tensors and arrays. ``T`` is the ``value_type`` of the container and ``A`` its ``allocator_type``. - ``XTENSOR_DEFAULT_SHAPE_CONTAINER(T, EA, SA)``: defines the type used as the default shape container for tensors and arrays. diff --git a/docs/source/expression.rst b/docs/source/expression.rst index 5d4bef4ce..67a80de02 100644 --- a/docs/source/expression.rst +++ b/docs/source/expression.rst @@ -137,7 +137,7 @@ You can access the elements of any ``xexpression`` with ``operator()``: It is possible to call ``operator()`` with fewer or more arguments than the number of dimensions of the expression: -- if ``operator()`` is called with too many arguments, we drops the most left ones +- if ``operator()`` is called with too many arguments, we drop the most left ones - if ``operator()`` is called with too few arguments, we prepend them with ``0`` values until we match the number of dimensions diff --git a/docs/source/histogram.rst b/docs/source/histogram.rst index f42b6a7d5..ea5f07f9a 100644 --- a/docs/source/histogram.rst +++ b/docs/source/histogram.rst @@ -80,4 +80,4 @@ The following algorithms are available: * ``logspace``: bins that logarithmically increase in size. -* ``uniform``: bin-edges such that the number of data-points is the same in all bins (as much as possible). +* ``uniform``: bin-edges such that the number of data points is the same in all bins (as much as possible). diff --git a/docs/source/indices.rst b/docs/source/indices.rst index 19683a4f6..a4f1a0af1 100644 --- a/docs/source/indices.rst +++ b/docs/source/indices.rst @@ -71,7 +71,7 @@ To print the ``std::vector``, it is converted to a ``xt::xtensor`` ar From array indices to flat indices ---------------------------------- -To convert the array indices to a ``xt::xtensor`` of flat indices, ``xt::ravel_indices`` can be used. For to same example: +To convert the array indices to a ``xt::xtensor`` of flat indices, ``xt::ravel_indices`` can be used. For the same example: .. code-block:: cpp @@ -126,7 +126,7 @@ For 1-D arrays the array indices and flat indices coincide. One can use the gene std::cout << xt::view(a, xt::keep(idx)) << std::endl; } -which print the indices and the selection (which are in this case identical): +which prints the indices and the selection (which are in this case identical): .. code-block:: none diff --git a/docs/source/view.rst b/docs/source/view.rst index 1d280e1bd..6f2144600 100644 --- a/docs/source/view.rst +++ b/docs/source/view.rst @@ -159,7 +159,7 @@ The ``xstrided_view`` is very efficient on contigous memory (e.g. ``xtensor`` or Transposed views ---------------- -``xtensor`` provides a lazy transposed view on any expression, whose layout is either row major order or column major order. Trying to build +``xtensor`` provides a lazy transposed view on any expression, whose layout is either row-major order or column major order. Trying to build a transposed view on a expression with a dynamic layout throws an exception. .. code:: From abf65127891b0f9574d3522e2ad607ba22441219 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Tue, 8 Sep 2020 12:27:01 +0200 Subject: [PATCH 094/606] Fixing various compiler warnings --- include/xtensor/xaxis_slice_iterator.hpp | 4 +-- include/xtensor/xfile_array.hpp | 10 +++---- include/xtensor/xmath.hpp | 35 ++++++++++++++++-------- include/xtensor/xset_operation.hpp | 4 +-- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/include/xtensor/xaxis_slice_iterator.hpp b/include/xtensor/xaxis_slice_iterator.hpp index 1c8d108e8..0329a767f 100644 --- a/include/xtensor/xaxis_slice_iterator.hpp +++ b/include/xtensor/xaxis_slice_iterator.hpp @@ -285,7 +285,7 @@ namespace xt * * @param e the expession to iterate over * @param axis the axis to iterate over - * @return an instance of xaxis_slice_iterator + * @return an instance of xaxis_slice_iterator */ template inline auto axis_slice_begin(E&& e, typename std::decay_t::size_type axis) @@ -317,7 +317,7 @@ namespace xt * * @param e the expession to iterate over * @param axis the axis to iterate over - * @return an instance of xaxis_slice_iterator + * @return an instance of xaxis_slice_iterator */ template inline auto axis_slice_end(E&& e, typename std::decay_t::size_type axis) diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index aa491dc76..ee7cde766 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -298,7 +298,7 @@ namespace xt { using is_stored = std::false_type; - static const char* path(const xexpression& e) + static const char* path(const xexpression&) { return ""; } @@ -320,7 +320,7 @@ namespace xt } template - constexpr bool is_stored(const xexpression& e) + constexpr bool is_stored(const xexpression&) { using return_type = typename detail::file_helper::is_stored; return return_type::value; @@ -429,7 +429,7 @@ namespace xt template template - inline auto xfile_array_container::operator()(Idxs... idxs) -> reference + inline auto xfile_array_container::operator()(Idxs... idxs) -> reference { return reference(m_storage(idxs...), m_dirty); } @@ -462,7 +462,7 @@ namespace xt } template - inline auto xfile_array_container::storage() const noexcept -> const storage_type& + inline auto xfile_array_container::storage() const noexcept -> const storage_type& { return m_storage; } @@ -514,7 +514,7 @@ namespace xt } template - inline auto xfile_array_container::data_element(size_type i) -> reference + inline auto xfile_array_container::data_element(size_type i) -> reference { return reference(m_storage.data_element(i), m_dirty); } diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index 88a39409c..f4a3f5e84 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -1882,6 +1882,9 @@ namespace detail { * \em axes. * @param e an \ref xexpression * @param axes the axes along which the product is computed (optional) + * @param ddof delta degrees of freedom (optional). + * The divisor used in calculations is N - ddof, where N represents the number of + elements. By default ddof is zero. * @param es evaluation strategy of the reducer * @return an \ref xreducer */ @@ -1909,26 +1912,32 @@ namespace detail { { // sum cannot always be a double. It could be a complex number which cannot operate on // std::plus. - const auto size = e.size(); + using size_type = typename std::decay_t::size_type; + const size_type size = e.size(); + XTENSOR_ASSERT(static_cast(ddof) <= size); auto s = sum(std::forward(e), std::forward(axes), es); - return mean_division(std::move(s), size - ddof); + return mean_division(std::move(s), size - static_cast(ddof)); } #ifdef X_OLD_CLANG template inline auto mean(E&& e, std::initializer_list axes, const D& ddof, EVS es) { - const auto size = e.size(); + using size_type = typename std::decay_t::size_type; + const size_type size = e.size(); + XTENSOR_ASSERT(static_cast(ddof) <= size); auto s = sum(std::forward(e), axes, es); - return detail::mean_division(std::move(s), size - ddof); + return detail::mean_division(std::move(s), size - static_cast(ddof)); } #else template inline auto mean(E&& e, const I (&axes)[N], const D& ddof, EVS es) { - const auto size = e.size(); + using size_type = typename std::decay_t::size_type; + const size_type size = e.size(); + XTENSOR_ASSERT(static_cast(ddof) <= size); auto s = sum(std::forward(e), axes, es); - return detail::mean_division(std::move(s), size - ddof); + return detail::mean_division(std::move(s), size - static_cast(ddof)); } #endif @@ -2116,7 +2125,9 @@ namespace detail { * * @param e an \ref xexpression * @param axes the axes along which the variance is computed (optional) - * @param ddof delta degrees of freedom (optional) + * @param ddof delta degrees of freedom (optional). + * The divisor used in calculations is N - ddof, where N represents the number of + elements. By default ddof is zero. * @param es evaluation strategy to use (lazy (default), or immediate) * @return an \ref xexpression * @@ -2947,13 +2958,13 @@ namespace detail { template inline auto interp(const E1 &x, const E2 &xp, const E3 &fp, T left, T right) { - using size_type = common_size_type_t; + using size_type = common_size_type_t; using value_type = typename E3::value_type; // basic checks - XTENSOR_ASSERT( xp.dimension() == 1 ); - XTENSOR_ASSERT( std::is_sorted(x.cbegin(), x.cend()) ); - XTENSOR_ASSERT( std::is_sorted(xp.cbegin(), xp.cend()) ); + XTENSOR_ASSERT(xp.dimension() == 1); + XTENSOR_ASSERT(std::is_sorted(x.cbegin(), x.cend())); + XTENSOR_ASSERT(std::is_sorted(xp.cbegin(), xp.cend())); // allocate output auto f = xtensor::from_shape(x.shape()); @@ -3055,7 +3066,7 @@ namespace detail { return covar; } - XTENSOR_ASSERT( x.dimension() == 2 ); + XTENSOR_ASSERT(x.dimension() == 2); auto covar = eval(zeros({ s[0], s[0] })); auto m = eval(mean(x, {1})); diff --git a/include/xtensor/xset_operation.hpp b/include/xtensor/xset_operation.hpp index 3addadb5b..4d51fbf98 100644 --- a/include/xtensor/xset_operation.hpp +++ b/include/xtensor/xset_operation.hpp @@ -178,14 +178,14 @@ namespace xt { for (size_t i = 0; i < v.size(); ++i) { - out(i) = std::lower_bound(a.cbegin(), a.cend(), v(i)) - a.cbegin(); + out(i) = static_cast(std::lower_bound(a.cbegin(), a.cend(), v(i)) - a.cbegin()); } } else { for (size_t i = 0; i < v.size(); ++i) { - out(i) = std::upper_bound(a.cbegin(), a.cend(), v(i)) - a.cbegin(); + out(i) = static_cast(std::upper_bound(a.cbegin(), a.cend(), v(i)) - a.cbegin()); } } From 540843802f16e15f4bb636be2de661c472fc5e42 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Mon, 7 Sep 2020 19:19:59 +0200 Subject: [PATCH 095/606] Adding rank member to allow sfinae based on rank --- docs/source/index.rst | 1 + docs/source/sfinae.rst | 77 +++++++++++++++++++++++++++++++ include/xtensor/xarray.hpp | 1 + include/xtensor/xfixed.hpp | 7 +-- include/xtensor/xtensor.hpp | 3 +- include/xtensor/xutils.hpp | 13 ++++++ include/xtensor/xview.hpp | 1 + test/CMakeLists.txt | 1 + test/test_sfinae.cpp | 90 +++++++++++++++++++++++++++++++++++++ 9 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 docs/source/sfinae.rst create mode 100644 test/test_sfinae.cpp diff --git a/docs/source/index.rst b/docs/source/index.rst index 2a9d4b34d..f8556bc70 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -69,6 +69,7 @@ for details. missing histogram random + sfinae file_loading build-options pitfall diff --git a/docs/source/sfinae.rst b/docs/source/sfinae.rst new file mode 100644 index 000000000..4cfc4a254 --- /dev/null +++ b/docs/source/sfinae.rst @@ -0,0 +1,77 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +.. _histogram: + +SFINAE +====== + +Rank overload +------------- + +All `xtensor`'s classes have a member ``rank`` that can be used +to overload based on rank using *SFINAE*. +Consider the following example: + +.. code-block:: cpp + + template ::value, int> = 0> + inline E foo(E&& a) + { + ... // act on object of flexible rank, or fixed rank != 2 + } + + template ::value, int> = 0> + inline E foo(E&& a) + { + ... // act on object of fixed rank == 2 + } + + TEST(sfinae, rank_basic) + { + xt::xarray a = {{9, 9}, {9, 9}}; + xt::xtensor b = {9, 9}; + xt::xtensor c = {{9, 9}, {9, 9}}; + + foo(a); // flexible rank -> first overload + foo(b); // fixed rank == 2 -> first overload + foo(c); // fixed rank == 2 -> second overload + } + +.. note:: + + If one wants to test for more than a single value for ``rank``, + one can use the default value ``SIZE_MAX`` used for flexible rank objects. + For example, one could have the following overloads: + + .. code-block:: cpp + + // flexible rank + template ::value, int> = 0> + inline E foo(E&& a); + + // fixed rank == 1 + template ::value, int> = 0> + inline E foo(E&& a); + + // fixed rank == 2 + template ::value, int> = 0> + inline E foo(E&& a); + + Note that fixed ranks other than 1 and 2 will raise a compiler error. + + Of course, if one wants a more limited scope, one could also do the following: + + .. code-block:: cpp + + // flexible rank + inline void foo(xt::xarray& a); + + // fixed rank == 1 + inline void foo(xt::xtensor& a); + + // fixed rank == 2 + inline void foo(xt::xtensor& a); diff --git a/include/xtensor/xarray.hpp b/include/xtensor/xarray.hpp index 69bdb474e..173c70ce7 100644 --- a/include/xtensor/xarray.hpp +++ b/include/xtensor/xarray.hpp @@ -104,6 +104,7 @@ namespace xt using inner_backstrides_type = typename base_type::inner_backstrides_type; using temporary_type = typename semantic_base::temporary_type; using expression_tag = Tag; + constexpr static std::size_t rank = SIZE_MAX; xarray_container(); explicit xarray_container(const shape_type& shape, layout_type l = L); diff --git a/include/xtensor/xfixed.hpp b/include/xtensor/xfixed.hpp index 26765f6ed..7835f9ea6 100644 --- a/include/xtensor/xfixed.hpp +++ b/include/xtensor/xfixed.hpp @@ -54,7 +54,7 @@ namespace xt namespace detail { /************************************************************************************** - The following is something we can currently only dream about -- for when we drop + The following is something we can currently only dream about -- for when we drop support for a lot of the old compilers (e.g. GCC 4.9, MSVC 2017 ;) template @@ -281,7 +281,7 @@ namespace xt * with tensor semantic and fixed dimension * * @tparam ET The type of the elements. - * @tparam S The xshape template paramter of the container. + * @tparam S The xshape template paramter of the container. * @tparam L The layout_type of the tensor. * @tparam SH Wether the tensor can be used as a shared expression. * @tparam Tag The expression tag. @@ -313,6 +313,7 @@ namespace xt using expression_tag = Tag; constexpr static std::size_t N = std::tuple_size::value; + constexpr static std::size_t rank = N; xfixed_container() = default; xfixed_container(const value_type& v); @@ -616,7 +617,7 @@ namespace xt /** * Allocates an xfixed_container with shape S with values from a C array. - * The type returned by get_init_type_t is raw C array ``value_type[X][Y][Z]`` for ``xt::xshape``. + * The type returned by get_init_type_t is raw C array ``value_type[X][Y][Z]`` for ``xt::xshape``. * C arrays can be initialized with the initializer list syntax, but the size is checked at compile * time to prevent errors. * Note: for clang < 3.8 this is an initializer_list and the size is not checked at compile-or runtime. diff --git a/include/xtensor/xtensor.hpp b/include/xtensor/xtensor.hpp index 8f347e328..483545eb0 100644 --- a/include/xtensor/xtensor.hpp +++ b/include/xtensor/xtensor.hpp @@ -105,6 +105,7 @@ namespace xt using inner_strides_type = typename base_type::inner_strides_type; using temporary_type = typename semantic_base::temporary_type; using expression_tag = Tag; + constexpr static std::size_t rank = N; xtensor_container(); xtensor_container(nested_initializer_list_t t); @@ -759,7 +760,7 @@ namespace xt std::fill(m_storage.begin(), m_storage.end(), e); return *this; } - + template inline auto xtensor_view::storage_impl() noexcept -> storage_type& { diff --git a/include/xtensor/xutils.hpp b/include/xtensor/xutils.hpp index fda1b8395..d0ddda942 100644 --- a/include/xtensor/xutils.hpp +++ b/include/xtensor/xutils.hpp @@ -857,6 +857,19 @@ namespace xt template using inner_reference_t = typename inner_reference::type; + + /************ + * has_rank * + ************/ + + template + struct has_rank + { + using type = std::integral_constant::rank == N>; + }; + + template + using has_rank_t = typename has_rank, N>::type; } #endif diff --git a/include/xtensor/xview.hpp b/include/xtensor/xview.hpp index 911b84d05..3aee09fc9 100644 --- a/include/xtensor/xview.hpp +++ b/include/xtensor/xview.hpp @@ -432,6 +432,7 @@ namespace xt using container_iterator = pointer; using const_container_iterator = const_pointer; + constexpr static std::size_t rank = SIZE_MAX; // The FSL argument prevents the compiler from calling this constructor // instead of the copy constructor when sizeof...(SL) == 0. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e64618b14..9c15c1ea2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -224,6 +224,7 @@ set(XTENSOR_TESTS test_extended_xhistogram.cpp test_extended_xsort.cpp test_xchunked_array.cpp + test_sfinae.cpp ) if(nlohmann_json_FOUND) diff --git a/test/test_sfinae.cpp b/test/test_sfinae.cpp new file mode 100644 index 000000000..b08455929 --- /dev/null +++ b/test/test_sfinae.cpp @@ -0,0 +1,90 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include +#include + +#include "gtest/gtest.h" +#include "xtensor/xtensor.hpp" +#include "xtensor/xarray.hpp" +// #include "xtensor/xfixed.hpp" +#include "xtensor/xview.hpp" + +namespace xt +{ + template ::value, int> = 0> + inline E sfinae_rank_basic_func(E&& a) + { + E b = a; + b.fill(0); + return b; + } + + template ::value, int> = 0> + inline E sfinae_rank_basic_func(E&& a) + { + E b = a; + b.fill(2); + return b; + } + + TEST(sfinae, rank_basic) + { + xt::xarray a = {{9, 9, 9}, {9, 9, 9}}; + xt::xtensor b = {9, 9}; + xt::xtensor c = {{9, 9}, {9, 9}}; + // xt::xtensor_fixed> d = {{9, 9}, {9, 9}}; + auto v = xt::view(c, 0, xt::all()); + + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(a), 0ul))); + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(b), 0ul))); + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(c), 2ul))); + // EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(d), 2ul))); + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(v), 0ul))); + } + + template ::value, int> = 0> + inline E sfinae_rank_func(E&& a) + { + E b = a; + b.fill(0); + return b; + } + + template ::value, int> = 0> + inline E sfinae_rank_func(E&& a) + { + E b = a; + b.fill(1); + return b; + } + + template ::value, int> = 0> + inline E sfinae_rank_func(E&& a) + { + E b = a; + b.fill(2); + return b; + } + + TEST(sfinae, rank) + { + xt::xarray a = {{9, 9, 9}, {9, 9, 9}}; + xt::xtensor b = {9, 9}; + xt::xtensor c = {{9, 9}, {9, 9}}; + // xt::xtensor_fixed> d = {{9, 9}, {9, 9}}; + auto v = xt::view(c, 0, xt::all()); + + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(a), 0ul))); + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(b), 1ul))); + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(c), 2ul))); + // EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(d), 2ul))); + EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(v), 0ul))); + } +} From d0b55c294882cadef3dcd3f81b2c9b5248a527e4 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 8 Sep 2020 18:03:46 +0200 Subject: [PATCH 096/606] Remove leftover --- include/xtensor/xchunk_store_manager.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 9de768e65..c374da60d 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -160,7 +160,6 @@ namespace xt fname.append(std::to_string(*it)); } path = m_directory + fname; - std::cout << path << std::endl; } /*************************************** From 62897f6744c191ce12ffdb27d78b56c525d52a85 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 8 Sep 2020 09:30:22 +0200 Subject: [PATCH 097/606] Removed conversion and initialization warnings --- include/xtensor/xaxis_slice_iterator.hpp | 2 +- include/xtensor/xchunk_store_manager.hpp | 4 ++-- include/xtensor/xchunked_array.hpp | 4 ++-- include/xtensor/xslice.hpp | 4 ++-- test/test_xmath_result_type.cpp | 1 + test/test_xnan_functions.cpp | 8 ++++++++ test/test_xreducer.cpp | 25 +++++++++++++----------- test/test_xview.cpp | 12 ++++++++++++ 8 files changed, 42 insertions(+), 18 deletions(-) diff --git a/include/xtensor/xaxis_slice_iterator.hpp b/include/xtensor/xaxis_slice_iterator.hpp index 0329a767f..3f90d05c2 100644 --- a/include/xtensor/xaxis_slice_iterator.hpp +++ b/include/xtensor/xaxis_slice_iterator.hpp @@ -144,7 +144,7 @@ namespace xt template inline xaxis_slice_iterator::xaxis_slice_iterator(CTA&& e, size_type axis, size_type index, size_type offset) : p_expression(get_storage_init(std::forward(e))), m_index(index), - m_offset(offset), m_axis_stride(e.strides()[axis] * (e.shape()[axis] - 1)), + m_offset(offset), m_axis_stride(static_cast(e.strides()[axis]) * (e.shape()[axis] - 1u)), m_lower_shape(0), m_upper_shape(0), m_iter_size(0), m_is_target_axis(false), m_sv(strided_view(std::forward(e), std::forward({ e.shape()[axis] }), std::forward({ e.strides()[axis] }), offset, e.layout())) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index c374da60d..667bd18d2 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -320,7 +320,7 @@ namespace xt std::size_t i; if (it1 != m_index_pool.cend()) { - i = std::distance(m_index_pool.cbegin(), it1); + i = static_cast(std::distance(m_index_pool.cbegin(), it1)); return m_chunk_pool[i]; } // if not, find a free chunk in the pool @@ -328,7 +328,7 @@ namespace xt const auto it2 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), empty_index); if (it2 != m_index_pool.cend()) { - i = std::distance(m_index_pool.cbegin(), it2); + i = static_cast(std::distance(m_index_pool.cbegin(), it2)); m_chunk_pool[i].set_path(path); m_index_pool[i].resize(static_cast(std::distance(first, last))); std::copy(first, last, m_index_pool[i].begin()); diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 624ec8f83..73c09cb81 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -427,8 +427,8 @@ namespace xt template inline std::pair xchunked_array::get_chunk_indexes_in_dimension(size_t dim, Idx idx) const { - size_t index_of_chunk = idx / m_chunk_shape[dim]; - size_t index_in_chunk = idx - index_of_chunk * m_chunk_shape[dim]; + size_t index_of_chunk = static_cast(idx) / m_chunk_shape[dim]; + size_t index_in_chunk = static_cast(idx) - index_of_chunk * m_chunk_shape[dim]; return std::make_pair(index_of_chunk, index_in_chunk); } diff --git a/include/xtensor/xslice.hpp b/include/xtensor/xslice.hpp index 2c34c100f..17eb55fa7 100644 --- a/include/xtensor/xslice.hpp +++ b/include/xtensor/xslice.hpp @@ -1374,7 +1374,7 @@ namespace xt std::size_t sz = m_indices.size(); for (std::size_t i = 0; i < sz; ++i) { - m_indices[i] = m_raw_indices[i] < 0 ? static_cast(shape) + m_raw_indices[i] : m_raw_indices[i]; + m_indices[i] = m_raw_indices[i] < 0 ? static_cast(shape) + m_raw_indices[i] : m_raw_indices[i]; } } @@ -1495,7 +1495,7 @@ namespace xt std::size_t sz = m_indices.size(); for (std::size_t i = 0; i < sz; ++i) { - m_indices[i] = m_raw_indices[i] < 0 ? static_cast(shape) + m_raw_indices[i] : m_raw_indices[i]; + m_indices[i] = m_raw_indices[i] < 0 ? static_cast(shape) + m_raw_indices[i] : m_raw_indices[i]; } size_type cum = size_type(0); size_type prev_cum = cum; diff --git a/test/test_xmath_result_type.cpp b/test/test_xmath_result_type.cpp index afcea8dcd..e5fe335da 100644 --- a/test/test_xmath_result_type.cpp +++ b/test/test_xmath_result_type.cpp @@ -19,6 +19,7 @@ #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wfloat-conversion" #pragma GCC diagnostic ignored "-Wsign-conversion" #include "xtensor/xarray.hpp" #include "xtensor/xmath.hpp" diff --git a/test/test_xnan_functions.cpp b/test/test_xnan_functions.cpp index ded72ea60..b9c532d86 100644 --- a/test/test_xnan_functions.cpp +++ b/test/test_xnan_functions.cpp @@ -8,6 +8,14 @@ ****************************************************************************/ #include "gtest/gtest.h" + +#if (defined(__GNUC__) && !defined(__clang__)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#include "xtensor/xmath.hpp" +#pragma GCC diagnostic pop +#endif + #include "xtensor/xarray.hpp" #include "xtensor/xtensor.hpp" #include "xtensor/xmath.hpp" diff --git a/test/test_xreducer.cpp b/test/test_xreducer.cpp index 0e25f2197..ea82083d9 100644 --- a/test/test_xreducer.cpp +++ b/test/test_xreducer.cpp @@ -9,23 +9,25 @@ #include "gtest/gtest.h" #include "test_common_macros.hpp" -#include "xtensor/xarray.hpp" -#include "xtensor/xtensor.hpp" +#if (defined(__GNUC__) && !defined(__clang__)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wfloat-conversion" +#include "xtensor/xmath.hpp" +#pragma GCC diagnostic pop +#else +#include "xtensor/xmath.hpp" +#endif #include "xtensor/xutils.hpp" #include "xtensor/xfixed.hpp" #include "xtensor/xbuilder.hpp" -#include "xtensor/xmath.hpp" #include "xtensor/xreducer.hpp" #include "xtensor/xview.hpp" #include "xtensor/xmanipulation.hpp" -#if (defined(__GNUC__) && !defined(__clang__)) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#include "xtensor/xrandom.hpp" -#pragma GCC diagnostic pop -#else +#include "xtensor/xarray.hpp" +#include "xtensor/xtensor.hpp" #include "xtensor/xrandom.hpp" -#endif #include "xtensor/xio.hpp" @@ -680,6 +682,7 @@ namespace xt xt::xtensor_fixed> a = {1, 2, 3}, b = {1, 2, 3}; xt::xtensor>, 1> c = {a, b}; auto res = xt::sum(c)(); - EXPECT_EQ(res, a * 2); + EXPECT_EQ(res, a * 2.); } } + diff --git a/test/test_xview.cpp b/test/test_xview.cpp index 9120ea983..f0deda474 100644 --- a/test/test_xview.cpp +++ b/test/test_xview.cpp @@ -11,6 +11,18 @@ #include "gtest/gtest.h" #include "test_common_macros.hpp" + +// Workaround to avoid warnings regarding initialization +// of distribution internal variables +#if (defined(__GNUC__) && !defined(__clang__)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#include "xtensor/xgenerator.hpp" +#pragma GCC diagnostic pop +#else +#endif + + #include "xtensor/xarray.hpp" #include "xtensor/xbuilder.hpp" #include "xtensor/xfixed.hpp" From 0934bafc73a7ab85f4d89b4ffaae3bb7cecb059a Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 11 Sep 2020 18:26:38 +0200 Subject: [PATCH 098/606] zfiles namespace renamed --- include/xtensor/zarray.hpp | 2 +- include/xtensor/zarray_impl.hpp | 2 +- include/xtensor/zassign.hpp | 2 +- include/xtensor/zdispatcher.hpp | 2 +- include/xtensor/zfunction.hpp | 2 +- include/xtensor/zmath.hpp | 2 +- test/CMakeLists.txt | 1 - test/test_zarray.cpp | 3 ++- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 8e299bc7b..2b07fb0ad 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -18,7 +18,7 @@ #include "zarray_impl.hpp" #include "zassign.hpp" -namespace xt +namespace zexperience { /********** diff --git a/include/xtensor/zarray_impl.hpp b/include/xtensor/zarray_impl.hpp index d845f8e0b..829671789 100644 --- a/include/xtensor/zarray_impl.hpp +++ b/include/xtensor/zarray_impl.hpp @@ -12,7 +12,7 @@ #include "xarray.hpp" -namespace xt +namespace zexperience { /************************* diff --git a/include/xtensor/zassign.hpp b/include/xtensor/zassign.hpp index 36ca622a4..3546dbd4b 100644 --- a/include/xtensor/zassign.hpp +++ b/include/xtensor/zassign.hpp @@ -13,7 +13,7 @@ #include "xassign.hpp" #include "zarray_impl.hpp" -namespace xt +namespace zexperience { template <> class xexpression_assigner diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp index 70cd756e7..04e9573a4 100644 --- a/include/xtensor/zdispatcher.hpp +++ b/include/xtensor/zdispatcher.hpp @@ -14,7 +14,7 @@ #include "zmath.hpp" -namespace xt +namespace zexperience { namespace mpl = xtl::mpl; diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp index 0f6fb55ea..046f68bc9 100644 --- a/include/xtensor/zfunction.hpp +++ b/include/xtensor/zfunction.hpp @@ -15,7 +15,7 @@ #include "zdispatcher.hpp" -namespace xt +namespace zexperience { template class zfunction : public xexpression> diff --git a/include/xtensor/zmath.hpp b/include/xtensor/zmath.hpp index 24f1ff1b9..47a598982 100644 --- a/include/xtensor/zmath.hpp +++ b/include/xtensor/zmath.hpp @@ -13,7 +13,7 @@ #include "xmath.hpp" #include "zarray_impl.hpp" -namespace xt +namespace zexperience { namespace detail { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9c15c1ea2..cb96b6c22 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -181,7 +181,6 @@ set(COMMON_BASE test_xview.cpp test_xview_semantic.cpp test_xutils.cpp - test_zarray.cpp ) set(XTENSOR_TESTS diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index 9741af8d1..674e17c18 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -12,8 +12,9 @@ #include "xtensor/zfunction.hpp" #ifndef XTENSOR_DISABLE_EXCEPTIONS -namespace xt +namespace zexperience { + using namespace xt; TEST(zarray, value_semantics) { xarray a = {{1., 2.}, {3., 4.}}; From 9733efcfd4bd76115c822a03dd79fbea3bda6589 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Sat, 12 Sep 2020 01:35:06 +0200 Subject: [PATCH 099/606] Release 0.21.6 --- README.md | 3 +- docs/source/changelog.rst | 73 +++++++++++++++++++++++++++++- environment.yml | 2 +- include/xtensor/xtensor_config.hpp | 2 +- 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ac34b6cf1..10fe9f240 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,8 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| -| master | ^0.6.16 | ^7.4.8 | +| master | ^0.6.18 | ^7.4.8 | +| 0.21.6 | ^0.6.18 | ^7.4.8 | | 0.21.5 | ^0.6.12 | ^7.4.6 | | 0.21.4 | ^0.6.12 | ^7.4.6 | | 0.21.3 | ^0.6.9 | ^7.4.4 | diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index b456e6f28..46d9edd0d 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -7,6 +7,78 @@ Changelog ========= +0.21.6 +------ + +- Added implementation of ``isin`` and ``in1d`` + `#2021 `_ +- Wrote single include header + `#2031 `_ +- Added details for ``xt::random`` to docs + `#2043 `_ +- Added ``digitize``, ``searchsorted``, and ``bin_items`` + `#2037 `_ +- Fixed error with zero tensor size in ``xt::mean`` + `#2047 `_ +- Fixed initialization order in ``xfunction`` + `#2050 `_ +- ``adapt_smart_ptr`` overloads now accept STL-like container as shape + `#2052 `_ +- Added ``xchunked_array`` + `#2076 `_ +- ``xchunked_array`` inherits from ``xiterable`` + `#2082 `_ +- ``xchunked_array`` inherits from ``xcontainer_semantic`` + `#2083 `_ +- Fixed assignment operator of ``xchunked_array`` + `#2084 `_ +- Added constructors from ``xexpression`` and ``chunk_shape`` to ``xchunked_array`` + `#2087 `_ +- Fixed chunk layout + `#2091 `_ +- Copy constructor gets expression's chunk_shape if it is chunked + `#2092 `_ +- Replaced template parameter chunk_type with chunk_storage + `#2095 `_ +- Implemented on-disk chunked array + `#2096 `_ +- Implemented chunk pool in xchunk_store_manager + `#2099 `_ +- ``xfile_array`` is now an expression + `#2107 `_ +- ``xchunked_array`` code cleanup + `#2109 `_ +- ``xchunked_store_manager`` code cleanup + `#2110 `_ +- Refactored ``xfile_array`` + `#2117 `_ +- Added simd accessors to ``xfil_array_container`` + `#2118 `_ +- Abstracted file format through a formal class + `#2115 `_ +- Added ``xchunked_array`` extension template + `#2122 `_ +- Refactored ``xdisk_io_handler`` + `#2123 `_ +- Fixed exception for file write operation + `#2125 `_ +- Implemented ``zarray`` + `#2127 `_ +- Implemented the skeleton of the dynamic expression system + `#2129 `_ +- Implemented zfunctions, equivalent of xfunction for dynamic expression system + `#2130 `_ +- Implemented ``allocate_result`` in ``zfunction`` + `#2132 `_ +- Implemented assign mechanism for ``zarray`` + `#2133 `_ +- Added xindex_path to transform indexes into path + `#2131 `_ +- Fixing various compiler warnings + `#2145 `_ +- Removed conversion and initialization warnings + `#2141 `_ + 0.21.5 ------ @@ -35,7 +107,6 @@ Changelog - Initialized all members of ``xfunciton_cache_impl`` `#2026 `_ - 0.21.4 ------ diff --git a/environment.yml b/environment.yml index ce388ff0a..d3c2bcebc 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xtensor channels: - conda-forge dependencies: - - xtensor=0.21.5 + - xtensor=0.21.6 - xtensor-blas=0.17.2 - xeus-cling=0.8.1 - blas * *openblas" diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index 271cfd1a2..e19dd6100 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -12,7 +12,7 @@ #define XTENSOR_VERSION_MAJOR 0 #define XTENSOR_VERSION_MINOR 21 -#define XTENSOR_VERSION_PATCH 6-dev +#define XTENSOR_VERSION_PATCH 6 // DETECT 3.6 <= clang < 3.8 for compiler bug workaround. #ifdef __clang__ From b36068dfa3aa31c1c078c574e731ca50e25ed4c2 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Mon, 14 Sep 2020 05:44:57 +0200 Subject: [PATCH 100/606] Removed zheaders from single header --- CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 82581339c..fd04854e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -316,7 +316,13 @@ list(REMOVE_ITEM XTENSOR_SINGLE_INCLUDE xtensor/xexpression_holder.hpp xtensor/xjson.hpp xtensor/xmime.hpp - xtensor/xnpy.hpp) + xtensor/xnpy.hpp + xtensor/zarray.hpp + xtensor/zarray_impl.hpp + xtensor/zassign.hpp + xtensor/zdispatcher.hpp + xtensor/zfunction.hpp + xtensor/zmath.hpp) PREPEND(XTENSOR_SINGLE_INCLUDE "#include <" ${XTENSOR_SINGLE_INCLUDE}) POSTFIX(XTENSOR_SINGLE_INCLUDE ">" ${XTENSOR_SINGLE_INCLUDE}) From 15603d4f226ea244ef154724909eb5cc4cf0e998 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 17 Sep 2020 11:37:18 +0200 Subject: [PATCH 101/606] Adding get_rank and has_fixed_rank --- docs/source/sfinae.rst | 39 ++++++++++++- include/xtensor/xutils.hpp | 32 ++++++++++- test/test_sfinae.cpp | 112 +++++++++++++++++++++++++++---------- 3 files changed, 150 insertions(+), 33 deletions(-) diff --git a/docs/source/sfinae.rst b/docs/source/sfinae.rst index 4cfc4a254..bd53f3644 100644 --- a/docs/source/sfinae.rst +++ b/docs/source/sfinae.rst @@ -30,7 +30,7 @@ Consider the following example: ... // act on object of fixed rank == 2 } - TEST(sfinae, rank_basic) + int main() { xt::xarray a = {{9, 9}, {9, 9}}; xt::xtensor b = {9, 9}; @@ -39,6 +39,8 @@ Consider the following example: foo(a); // flexible rank -> first overload foo(b); // fixed rank == 2 -> first overload foo(c); // fixed rank == 2 -> second overload + + return 0; } .. note:: @@ -50,7 +52,7 @@ Consider the following example: .. code-block:: cpp // flexible rank - template ::value, int> = 0> + template ::value, int> = 0> inline E foo(E&& a); // fixed rank == 1 @@ -75,3 +77,36 @@ Consider the following example: // fixed rank == 2 inline void foo(xt::xtensor& a); + +Rank as member +-------------- + +If you want to use the rank as a member of your own class you can use ``xt::get_rank``. +Consider the following example: + +.. code-block:: cpp + + template + struct Foo + { + static const size_t rank = xt::get_rank::value; + + static size_t value() + { + return rank; + } + }; + + int main() + { + xt::xtensor A = xt::zeros({2}); + xt::xtensor B = xt::zeros({2, 2}); + xt::xarray C = xt::zeros({2, 2}); + + std::cout << Foo::value() << std::endl; + std::cout << Foo::value() << std::endl; + std::cout << Foo::value() << std::endl; + + return 0; + } + diff --git a/include/xtensor/xutils.hpp b/include/xtensor/xutils.hpp index d0ddda942..3fb4e59b5 100644 --- a/include/xtensor/xutils.hpp +++ b/include/xtensor/xutils.hpp @@ -858,6 +858,35 @@ namespace xt template using inner_reference_t = typename inner_reference::type; + /************ + * get_rank * + ************/ + + template + struct get_rank + { + constexpr static std::size_t value = SIZE_MAX; + }; + + template + struct get_rank + { + constexpr static std::size_t value= E::rank; + }; + + /****************** + * has_fixed_rank * + ******************/ + + template + struct has_fixed_rank + { + using type = std::integral_constant>::value != SIZE_MAX>; + }; + + template + using has_fixed_rank_t = typename has_fixed_rank>::type; + /************ * has_rank * ************/ @@ -865,11 +894,12 @@ namespace xt template struct has_rank { - using type = std::integral_constant::rank == N>; + using type = std::integral_constant>::value == N>; }; template using has_rank_t = typename has_rank, N>::type; + } #endif diff --git a/test/test_sfinae.cpp b/test/test_sfinae.cpp index b08455929..2f95909f4 100644 --- a/test/test_sfinae.cpp +++ b/test/test_sfinae.cpp @@ -19,19 +19,15 @@ namespace xt { template ::value, int> = 0> - inline E sfinae_rank_basic_func(E&& a) + inline size_t sfinae_rank_basic_func(E&&) { - E b = a; - b.fill(0); - return b; + return 0; } template ::value, int> = 0> - inline E sfinae_rank_basic_func(E&& a) + inline size_t sfinae_rank_basic_func(E&&) { - E b = a; - b.fill(2); - return b; + return 2; } TEST(sfinae, rank_basic) @@ -42,35 +38,33 @@ namespace xt // xt::xtensor_fixed> d = {{9, 9}, {9, 9}}; auto v = xt::view(c, 0, xt::all()); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(a), 0ul))); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(b), 0ul))); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(c), 2ul))); - // EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(d), 2ul))); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_basic_func(v), 0ul))); + EXPECT_TRUE(sfinae_rank_basic_func(a) == 0ul); + EXPECT_TRUE(sfinae_rank_basic_func(b) == 0ul); + EXPECT_TRUE(sfinae_rank_basic_func(c) == 2ul); + // EXPECT_TRUE(sfinae_rank_basic_func(d) == 2ul); + EXPECT_TRUE(sfinae_rank_basic_func(v) == 0ul); + EXPECT_TRUE(sfinae_rank_basic_func(2ul * a) == 0ul); + EXPECT_TRUE(sfinae_rank_basic_func(2ul * b) == 0ul); + EXPECT_TRUE(sfinae_rank_basic_func(2ul * c) == 0ul); + } template ::value, int> = 0> - inline E sfinae_rank_func(E&& a) + inline size_t sfinae_rank_func(E&&) { - E b = a; - b.fill(0); - return b; + return 0; } template ::value, int> = 0> - inline E sfinae_rank_func(E&& a) + inline size_t sfinae_rank_func(E&&) { - E b = a; - b.fill(1); - return b; + return 1; } template ::value, int> = 0> - inline E sfinae_rank_func(E&& a) + inline size_t sfinae_rank_func(E&&) { - E b = a; - b.fill(2); - return b; + return 2; } TEST(sfinae, rank) @@ -81,10 +75,68 @@ namespace xt // xt::xtensor_fixed> d = {{9, 9}, {9, 9}}; auto v = xt::view(c, 0, xt::all()); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(a), 0ul))); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(b), 1ul))); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(c), 2ul))); - // EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(d), 2ul))); - EXPECT_TRUE(xt::all(xt::equal(sfinae_rank_func(v), 0ul))); + EXPECT_TRUE(sfinae_rank_func(a) == 0ul); + EXPECT_TRUE(sfinae_rank_func(b) == 1ul); + EXPECT_TRUE(sfinae_rank_func(c) == 2ul); + // EXPECT_TRUE(sfinae_rank_func(d) == 2ul); + EXPECT_TRUE(sfinae_rank_func(v) == 0ul); + EXPECT_TRUE(sfinae_rank_func(2ul * a) == 0ul); + EXPECT_TRUE(sfinae_rank_func(2ul * b) == 0ul); + EXPECT_TRUE(sfinae_rank_func(2ul * c) == 0ul); + } + + template ::value, int> = 0> + inline bool sfinae_fixed_func(E&&) + { + return false; + } + + template ::value, int> = 0> + inline bool sfinae_fixed_func(E&&) + { + return true; + } + + TEST(sfinae, fixed_rank) + { + xt::xarray a = {{9, 9, 9}, {9, 9, 9}}; + xt::xtensor b = {9, 9}; + xt::xtensor c = {{9, 9}, {9, 9}}; + // xt::xtensor_fixed> d = {{9, 9}, {9, 9}}; + auto v = xt::view(c, 0, xt::all()); + + EXPECT_TRUE(sfinae_fixed_func(a) == false); + EXPECT_TRUE(sfinae_fixed_func(b) == true); + EXPECT_TRUE(sfinae_fixed_func(c) == true); + // EXPECT_TRUE(sfinae_fixed_func(d) == 2ul); + EXPECT_TRUE(sfinae_fixed_func(v) == false); + EXPECT_TRUE(sfinae_fixed_func(2ul * a) == false); + EXPECT_TRUE(sfinae_fixed_func(2ul * b) == false); + EXPECT_TRUE(sfinae_fixed_func(2ul * c) == false); + } + + template + struct sfinae_get_rank + { + static const size_t rank = xt::get_rank::value; + + static size_t value() + { + return rank; + } + }; + + TEST(sfinae, get_rank) + { + xt::xtensor A = xt::zeros({2}); + xt::xtensor B = xt::zeros({2, 2}); + xt::xarray C = xt::zeros({2, 2}); + + EXPECT_TRUE(sfinae_get_rank::value() == 1ul); + EXPECT_TRUE(sfinae_get_rank::value() == 2ul); + EXPECT_TRUE(sfinae_get_rank::value() == SIZE_MAX); + EXPECT_TRUE(sfinae_get_rank::value() == SIZE_MAX); + EXPECT_TRUE(sfinae_get_rank::value() == SIZE_MAX); + EXPECT_TRUE(sfinae_get_rank::value() == SIZE_MAX); } } From 0ec9001e723c9db861254e9b02b5fdc0530ef9a2 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 17 Sep 2020 11:54:27 +0200 Subject: [PATCH 102/606] Adding has_shape --- docs/source/api/xshape.rst | 19 ++++++++++++++++ include/xtensor/xshape.hpp | 45 +++++++++++++++++++++++++++++++++++++- test/test_xshape.cpp | 8 +++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 docs/source/api/xshape.rst diff --git a/docs/source/api/xshape.rst b/docs/source/api/xshape.rst new file mode 100644 index 000000000..62fb6a280 --- /dev/null +++ b/docs/source/api/xshape.rst @@ -0,0 +1,19 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +xshape +====== + +Defined in ``xtensor/xshape.hpp`` + +.. doxygenfunction:: bool same_shape(const S1& s1, const S2& s2) + :project: xtensor + +.. doxygenfunction:: bool has_shape(const E& e, std::initializer_list shape) + :project: xtensor + +.. doxygenfunction:: bool has_shape(const E& e, const S& shape) + :project: xtensor diff --git a/include/xtensor/xshape.hpp b/include/xtensor/xshape.hpp index adfedb218..e9ee57826 100644 --- a/include/xtensor/xshape.hpp +++ b/include/xtensor/xshape.hpp @@ -35,7 +35,7 @@ namespace xt class fixed_shape; using xindex = dynamic_shape; - + template bool same_shape(const S1& s1, const S2& s2) noexcept; @@ -96,12 +96,55 @@ namespace xt * same_shape * **************/ + /** + * @ingroup same_shape + * @brief same_shape + * + * Check if two objects have the same shape. + * @param s1 an array + * @param s2 an array + * @return bool + */ template inline bool same_shape(const S1& s1, const S2& s2) noexcept { return s1.size() == s2.size() && std::equal(s1.begin(), s1.end(), s2.begin()); } + /************* + * has_shape * + *************/ + + /** + * @ingroup has_shape + * @brief has_shape + * + * Check if an object has a certain shape. + * @param a an array + * @param shape the shape to test + * @return bool + */ + template + inline bool has_shape(const E& e, std::initializer_list shape) noexcept + { + return e.shape().size() == shape.size() && std::equal(e.shape().cbegin(), e.shape().cend(), shape.begin()); + } + + /** + * @ingroup has_shape + * @brief has_shape + * + * Check if an object has a certain shape. + * @param a an array + * @param shape the shape to test + * @return bool + */ + template ::value>> + inline bool has_shape(const E& e, const S& shape) + { + return e.shape().size() == shape.size() && std::equal(e.shape().cbegin(), e.shape().cend(), shape.begin()); + } + /************************* * initializer_dimension * *************************/ diff --git a/test/test_xshape.cpp b/test/test_xshape.cpp index d62afc019..7cdc56eb9 100644 --- a/test/test_xshape.cpp +++ b/test/test_xshape.cpp @@ -58,4 +58,12 @@ namespace xt ASSERT_TRUE(expect_v); ASSERT_TRUE(expect_a); } + + TEST(xshape, has_shape) + { + std::array shape = {2, 3}; + xt::xtensor A = xt::zeros(shape); + ASSERT_TRUE(xt::has_shape(A, shape)); + ASSERT_TRUE(xt::has_shape(A, {2, 3})); + } } From 4790ffa905dcd2c18981880f1bb5afd0b6d8990d Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Sun, 20 Sep 2020 22:25:58 +0200 Subject: [PATCH 103/606] Implemented insertion of range and intializer list in svector --- include/xtensor/xstorage.hpp | 33 +++++++++++++++++++++++++++ test/test_xstorage.cpp | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/include/xtensor/xstorage.hpp b/include/xtensor/xstorage.hpp index c752159bc..5d9f8696e 100644 --- a/include/xtensor/xstorage.hpp +++ b/include/xtensor/xstorage.hpp @@ -715,6 +715,11 @@ namespace xt iterator insert(const_iterator it, const T& elt); + template + iterator insert(const_iterator pos, It first, It last); + + iterator insert(const_iterator pos, std::initializer_list l); + template void swap(svector& rhs); @@ -1183,6 +1188,34 @@ namespace xt return it; } + template + template + inline auto svector::insert(const_iterator pos, It first, It last) -> iterator + { + auto it = const_cast(pos); + difference_type n = std::distance(first, last); + if (n > 0) + { + if (n > m_capacity - m_end) + { + std::ptrdiff_t elt_no = it - m_begin; + grow(static_cast((m_capacity - m_begin) + n)); + it = m_begin + elt_no; + } + + std::move_backward(it, m_end, m_end + n); + m_end += n; + std::copy(first, last, it); + } + return it; + } + + template + inline auto svector::insert(const_iterator pos, std::initializer_list l) -> iterator + { + return insert(pos, l.begin(), l.end()); + } + template inline void svector::destroy_range(T* begin, T* end) { diff --git a/test/test_xstorage.cpp b/test/test_xstorage.cpp index 91a9f1ac8..54ac95e2a 100644 --- a/test/test_xstorage.cpp +++ b/test/test_xstorage.cpp @@ -144,6 +144,49 @@ namespace xt EXPECT_TRUE(std::equal(s.begin(), s.end(), v.begin())); } + TEST(svector, insert_range) + { + svector_type s0 = {1,2,3,4}; + svector_type s1 = s0; + svector_type s2 = s0; + std::vector v0(s0.begin(), s0.end()); + std::vector v1(v0); + std::vector v2(v0); + std::vector ins = {1, 4}; + + s0.insert(s0.begin(), ins.cbegin(), ins.cend()); + v0.insert(v0.begin(), ins.cbegin(), ins.cend()); + s1.insert(s1.begin()+2, ins.cbegin(), ins.cend()); + v1.insert(v1.begin()+2, ins.cbegin(), ins.cend()); + s2.insert(s2.begin()+4, ins.cbegin(), ins.cend()); + v2.insert(v2.begin()+4, ins.cbegin(), ins.cend()); + + EXPECT_TRUE(std::equal(s0.begin(), s0.end(), v0.begin())); + EXPECT_TRUE(std::equal(s1.begin(), s1.end(), v1.begin())); + EXPECT_TRUE(std::equal(s2.begin(), s2.end(), v2.begin())); + } + + TEST(svector, insert_initializer_list) + { + svector_type s0 = {1,2,3,4}; + svector_type s1 = s0; + svector_type s2 = s0; + std::vector v0(s0.begin(), s0.end()); + std::vector v1(v0); + std::vector v2(v0); + + s0.insert(s0.begin(), {1u, 4u}); + v0.insert(v0.begin(), {1u, 4u}); + s1.insert(s1.begin()+2, {1u, 4u}); + v1.insert(v1.begin()+2, {1u, 4u}); + s2.insert(s2.begin()+4, {1u, 4u}); + v2.insert(v2.begin()+4, {1u, 4u}); + + EXPECT_TRUE(std::equal(s0.begin(), s0.end(), v0.begin())); + EXPECT_TRUE(std::equal(s1.begin(), s1.end(), v1.begin())); + EXPECT_TRUE(std::equal(s2.begin(), s2.end(), v2.begin())); + } + TEST(svector, constructor) { svector_type a; From a2bf54dc5e9bc3f60fc17fcb8f8b9d44794a9015 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Mon, 7 Sep 2020 12:36:05 +0200 Subject: [PATCH 104/606] Added dispatching types lists and registration mechanism --- CMakeLists.txt | 9 +- include/xtensor/zarray.hpp | 2 +- include/xtensor/zarray_impl.hpp | 2 +- include/xtensor/zassign.hpp | 2 +- include/xtensor/zdispatcher.hpp | 123 ++++++++++++++++++++-- include/xtensor/zdispatching_types.hpp | 137 +++++++++++++++++++++++++ include/xtensor/zfunction.hpp | 2 +- include/xtensor/zmath.hpp | 7 +- test/CMakeLists.txt | 1 + test/test_zarray.cpp | 2 +- 10 files changed, 261 insertions(+), 26 deletions(-) create mode 100644 include/xtensor/zdispatching_types.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fd04854e1..ba8dc69f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -182,6 +182,7 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/zarray.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zarray_impl.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zdispatcher.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/zdispatching_types.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zfunction.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zmath.hpp ) @@ -316,13 +317,7 @@ list(REMOVE_ITEM XTENSOR_SINGLE_INCLUDE xtensor/xexpression_holder.hpp xtensor/xjson.hpp xtensor/xmime.hpp - xtensor/xnpy.hpp - xtensor/zarray.hpp - xtensor/zarray_impl.hpp - xtensor/zassign.hpp - xtensor/zdispatcher.hpp - xtensor/zfunction.hpp - xtensor/zmath.hpp) + xtensor/xnpy.hpp) PREPEND(XTENSOR_SINGLE_INCLUDE "#include <" ${XTENSOR_SINGLE_INCLUDE}) POSTFIX(XTENSOR_SINGLE_INCLUDE ">" ${XTENSOR_SINGLE_INCLUDE}) diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 2b07fb0ad..8e299bc7b 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -18,7 +18,7 @@ #include "zarray_impl.hpp" #include "zassign.hpp" -namespace zexperience +namespace xt { /********** diff --git a/include/xtensor/zarray_impl.hpp b/include/xtensor/zarray_impl.hpp index 829671789..d845f8e0b 100644 --- a/include/xtensor/zarray_impl.hpp +++ b/include/xtensor/zarray_impl.hpp @@ -12,7 +12,7 @@ #include "xarray.hpp" -namespace zexperience +namespace xt { /************************* diff --git a/include/xtensor/zassign.hpp b/include/xtensor/zassign.hpp index 3546dbd4b..36ca622a4 100644 --- a/include/xtensor/zassign.hpp +++ b/include/xtensor/zassign.hpp @@ -13,7 +13,7 @@ #include "xassign.hpp" #include "zarray_impl.hpp" -namespace zexperience +namespace xt { template <> class xexpression_assigner diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp index 04e9573a4..1b7a6752d 100644 --- a/include/xtensor/zdispatcher.hpp +++ b/include/xtensor/zdispatcher.hpp @@ -12,14 +12,13 @@ #include +#include "zdispatching_types.hpp" #include "zmath.hpp" -namespace zexperience +namespace xt { namespace mpl = xtl::mpl; - using supported_type = mpl::vector; - template using zrun_dispatcher_impl = xtl::functor_dispatcher < @@ -54,6 +53,9 @@ namespace zexperience template static void insert(); + template + static void register_dispatching(mpl::vector, U...>); + static void init(); static void dispatch(const zarray_impl& z1, zarray_impl& res); static size_t get_type_index(const zarray_impl& z1); @@ -68,13 +70,18 @@ namespace zexperience template void insert_impl(); + template + inline void register_dispatching_impl(mpl::vector, U...>); + inline void register_dispatching_impl(mpl::vector<>); + using zfunctor_type = get_zmapped_functor_t; using ztype_dispatcher = ztype_dispatcher_impl>; using zrun_dispatcher = zrun_dispatcher_impl>; ztype_dispatcher m_type_dispatcher; zrun_dispatcher m_run_dispatcher; - }; + }; + /********************** * ztriple_dispatcher * @@ -91,6 +98,9 @@ namespace zexperience template static void insert(); + template + static void register_dispatching(mpl::vector, U...>); + static void init(); static void dispatch(const zarray_impl& z1, const zarray_impl& z2, zarray_impl& res); static size_t get_type_index(const zarray_impl& z1, const zarray_impl& z2); @@ -105,6 +115,10 @@ namespace zexperience template void insert_impl(); + template + inline void register_dispatching_impl(mpl::vector, U...>); + inline void register_dispatching_impl(mpl::vector<>); + using zfunctor_type = get_zmapped_functor_t; using ztype_dispatcher = ztype_dispatcher_impl>; using zrun_dispatcher = zrun_dispatcher_impl>; @@ -180,6 +194,30 @@ namespace zexperience * zdouble_dispatcher implementation * *************************************/ + namespace detail + { + template + struct unary_dispatching_types + { + using type = zunary_func_types; + }; + + template <> + struct unary_dispatching_types + { + using type = zunary_op_types; + }; + + template <> + struct unary_dispatching_types + { + using type = zunary_op_types; + }; + + template + using unary_dispatching_types_t = typename unary_dispatching_types::type; + } + template template inline void zdouble_dispatcher::insert() @@ -187,6 +225,13 @@ namespace zexperience instance().template insert_impl(); } + template + template + inline void zdouble_dispatcher::register_dispatching(mpl::vector, U...>) + { + instance().register_dispatching_impl(mpl::vector, U...>()); + } + template inline void zdouble_dispatcher::init() { @@ -215,8 +260,7 @@ namespace zexperience template inline zdouble_dispatcher::zdouble_dispatcher() { - insert_impl(); - insert_impl(); + register_dispatching_impl(detail::unary_dispatching_types_t()); } template @@ -229,10 +273,49 @@ namespace zexperience m_type_dispatcher.template insert(&zfunctor_type::template index); } + template + template + inline void zdouble_dispatcher::register_dispatching_impl(mpl::vector, U...>) + { + insert_impl(); + register_dispatching_impl(mpl::vector()); + } + + template + inline void zdouble_dispatcher::register_dispatching_impl(mpl::vector<>) + { + } + /************************************* * ztriple_dispatcher implementation * *************************************/ + namespace detail + { + using zbinary_func_list = mpl::vector + < + math::atan2_fun, + math::hypot_fun, + math::pow_fun, + math::fdim_fun, + math::fmax_fun, + math::fmin_fun, + math::remainder_fun, + math::fmod_fun + >; + + template + struct binary_dispatching_types + { + using type = std::conditional_t::value, + zbinary_func_types, + zbinary_op_types>; + }; + + template + using binary_dispatching_types_t = typename binary_dispatching_types::type; + } + template template inline void ztriple_dispatcher::insert() @@ -240,6 +323,13 @@ namespace zexperience instance().template insert_impl(); } + template + template + inline void ztriple_dispatcher::register_dispatching(mpl::vector, U...>) + { + instance().register_impl(mpl::vector, U...>()); + } + template inline void ztriple_dispatcher::init() { @@ -268,8 +358,7 @@ namespace zexperience template inline ztriple_dispatcher::ztriple_dispatcher() { - insert_impl(); - insert_impl(); + register_dispatching_impl(detail::binary_dispatching_types_t()); } template @@ -283,6 +372,20 @@ namespace zexperience m_type_dispatcher.template insert(&zfunctor_type::template index); } + + template + template + inline void ztriple_dispatcher::register_dispatching_impl(mpl::vector, U...>) + { + insert_impl(); + register_dispatching_impl(mpl::vector()); + } + + template + inline void ztriple_dispatcher::register_dispatching_impl(mpl::vector<>) + { + } + /*************************************** * zarray_impl_register implementation * ***************************************/ @@ -404,7 +507,7 @@ namespace zexperience zdispatcher_t::init(); zdispatcher_t::init(); zdispatcher_t::init(); - zdispatcher_t::init(); + /*zdispatcher_t::init(); zdispatcher_t::init(); zdispatcher_t::init(); zdispatcher_t::init(); @@ -412,7 +515,7 @@ namespace zexperience zdispatcher_t::init(); zdispatcher_t::init(); zdispatcher_t::init(); - zdispatcher_t::init(); + zdispatcher_t::init();*/ } } diff --git a/include/xtensor/zdispatching_types.hpp b/include/xtensor/zdispatching_types.hpp new file mode 100644 index 000000000..e3afb85fb --- /dev/null +++ b/include/xtensor/zdispatching_types.hpp @@ -0,0 +1,137 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XTENSOR_ZDISPATCHING_TYPES_HPP +#define XTENSOR_ZDISPATCHING_TYPES_HPP + +#include + +namespace xt +{ + namespace mpl = xtl::mpl; + + // TODO: move to XTL + namespace detail + { + template + struct concatenate; + + template + struct concatenate, mpl::vector> + { + using type = mpl::vector; + }; + + template + struct concatenate, mpl::vector, L...> + { + using type = typename concatenate< + typename concatenate< + mpl::vector, + mpl::vector + >::type, + L... + >::type; + }; + + template + using concatenate_t = typename concatenate::type; + } + + /*********** + * z types * + ***********/ + + using z_int_types = mpl::vector; + using z_small_int_types = mpl::vector; + using z_big_int_types = mpl::vector; + using z_float_types = mpl::vector; + + using z_types = detail::concatenate_t; + + /************************* + * unary operation types * + *************************/ + + template + struct build_unary_impl + { + using type = mpl::vector; + }; + + template + using build_unary_impl_t = typename build_unary_impl::type; + + template + using build_unary_identity_t = build_unary_impl_t; + + template + using build_unary_int32_t = build_unary_impl_t; + + template + using build_unary_int64_t = build_unary_impl_t; + + template + using build_unary_double_t = build_unary_impl_t; + + using zunary_func_types = detail::concatenate_t< + mpl::transform_t, + mpl::transform_t + >; + + using zunary_op_types = detail::concatenate_t< + mpl::transform_t, + mpl::transform_t, + mpl::transform_t + >; + + /************************** + * binary operation types * + **************************/ + + template + struct build_binary_impl + { + using type = mpl::vector; + }; + + template + using build_binary_impl_t = typename build_binary_impl::type; + + template + using build_binary_identity_t = build_binary_impl_t; + + template + using build_binary_int32_t = build_binary_impl_t; + + template + using build_binary_int64_t = build_binary_impl_t; + + template + using build_binary_double_t = build_binary_impl_t; + + using zbinary_func_types = detail::concatenate_t< + mpl::transform_t, + mpl::transform_t + >; + + using zbinary_op_types = detail::concatenate_t< + mpl::transform_t, + mpl::transform_t, + mpl::transform_t + >; + +} + +#endif + diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp index 046f68bc9..0f6fb55ea 100644 --- a/include/xtensor/zfunction.hpp +++ b/include/xtensor/zfunction.hpp @@ -15,7 +15,7 @@ #include "zdispatcher.hpp" -namespace zexperience +namespace xt { template class zfunction : public xexpression> diff --git a/include/xtensor/zmath.hpp b/include/xtensor/zmath.hpp index 47a598982..d6fba67d0 100644 --- a/include/xtensor/zmath.hpp +++ b/include/xtensor/zmath.hpp @@ -13,7 +13,7 @@ #include "xmath.hpp" #include "zarray_impl.hpp" -namespace zexperience +namespace xt { namespace detail { @@ -55,7 +55,6 @@ namespace zexperience }; \ XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) -#define ZCONCAT(X, Y) X Y #define XTENSOR_BINARY_ZOPERATOR(ZNAME, XOP, XFUN) \ struct ZNAME \ { \ @@ -175,7 +174,7 @@ namespace zexperience XTENSOR_UNARY_ZFUNCTOR(zerfc, xt::erfc, math::erfc_fun); XTENSOR_UNARY_ZFUNCTOR(ztgamma, xt::tgamma, math::tgamma_fun); XTENSOR_UNARY_ZFUNCTOR(zlgamma, xt::lgamma, math::lgamma_fun); - XTENSOR_UNARY_ZFUNCTOR(zceil, xt::ceil, math::ceil_fun); + /*XTENSOR_UNARY_ZFUNCTOR(zceil, xt::ceil, math::ceil_fun); XTENSOR_UNARY_ZFUNCTOR(zfloor, xt::floor, math::floor_fun); XTENSOR_UNARY_ZFUNCTOR(ztrunc, xt::trunc, math::trunc_fun); XTENSOR_UNARY_ZFUNCTOR(zround, xt::round, math::round_fun); @@ -183,7 +182,7 @@ namespace zexperience XTENSOR_UNARY_ZFUNCTOR(zrint, xt::rint, math::rint_fun); XTENSOR_UNARY_ZFUNCTOR(zisfinite, xt::isfinite, math::isfinite_fun); XTENSOR_UNARY_ZFUNCTOR(zisinf, xt::isinf, math::isinf_fun); - XTENSOR_UNARY_ZFUNCTOR(zisnan, xt::isnan, math::isnan_fun); + XTENSOR_UNARY_ZFUNCTOR(zisnan, xt::isnan, math::isnan_fun);*/ #undef XTENSOR_BINARY_ZFUNCTOR #undef XTENSOR_UNARY_ZFUNCTOR diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cb96b6c22..9c15c1ea2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -181,6 +181,7 @@ set(COMMON_BASE test_xview.cpp test_xview_semantic.cpp test_xutils.cpp + test_zarray.cpp ) set(XTENSOR_TESTS diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index 674e17c18..133048725 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -12,7 +12,7 @@ #include "xtensor/zfunction.hpp" #ifndef XTENSOR_DISABLE_EXCEPTIONS -namespace zexperience +namespace xt { using namespace xt; TEST(zarray, value_semantics) From ffe8095a96fcd8ce329c9b03ba11e6dfdc04fe98 Mon Sep 17 00:00:00 2001 From: Wai-Shing Luk Date: Wed, 30 Sep 2020 05:54:57 +0000 Subject: [PATCH 105/606] fix for the issue #2168 --- include/xtensor/xtensor_config.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index e19dd6100..f7e75f9cc 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -31,6 +31,7 @@ // Exception support. #if defined(XTENSOR_DISABLE_EXCEPTIONS) +#include #define XTENSOR_THROW(_, msg) \ { \ std::cerr << msg << std::endl; \ From 481331475513290e5745a28d683e8bc573731f9c Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 1 Oct 2020 10:50:40 +0200 Subject: [PATCH 106/606] [docs] Extending docs random --- docs/source/random.rst | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/source/random.rst b/docs/source/random.rst index 9ed21c010..cbfec9fc4 100644 --- a/docs/source/random.rst +++ b/docs/source/random.rst @@ -70,7 +70,8 @@ xt::random::gamma :ref:`xt::random::gamma ` -Produces (an array of) random positive floating-point values, distributed according to the probability density: +Produces (an array of) random positive floating-point values, +distributed according to the probability density: .. math:: @@ -80,20 +81,49 @@ where :math:`\alpha` is the shape (also known as :math:`k`) and :math:`\beta` th .. note:: - Do not confuse the first argument of ``xt::random``, the shape of the output array, with the parameter :math:`alpha`. + Different from NumPy, the first argument is the shape of the output array. .. seealso:: * `numpy.random.gamma `_ * `std::gamma_distribution `_ * `Weisstein, Eric W. "Gamma Distribution." From MathWorld – A Wolfram Web Resource. `_ - * `Wikipedia, "Gamma distribution". `_ + * `Wikipedia, "Gamma distribution". `_ xt::random::weibull =================== :ref:`xt::random::weibull ` +Produces (an array of) random positive floating-point values, +distributed according to the probability density: + +.. math:: + + P(x) = \frac{a}{b} \left( \frac{x}{b} \right)^{a - 1} e^{-(x / b)^a} + +where :math:`a > 0` is the shape parameter and :math:`b > 0` the scale parameter. +In particular, a random variable is produced as + +.. math:: + + X = b (- \ln (U))^{1/a} + +where :math:`U` is drawn from the uniform distribution (0, 1]. + +By default both the shape :math:`a = 1` and the scale :math:`b = 1`. +Note that you can specify only :math:`a` while choosing the default for :math:`b`. + +.. note:: + + Different from NumPy, the first argument is the shape of the output array. + +.. seealso:: + + * `numpy.random.weibull `_ + * `std::weibull_distribution `_ + * `Wikipedia, "Weibull distribution". `_ + xt::random::extreme_value ========================= From 48a17067a6a8d52769f6fda7ae560b8a872892ef Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 2 Oct 2020 11:39:32 +0200 Subject: [PATCH 107/606] Release 0.21.7 --- README.md | 1 + docs/source/changelog.rst | 18 ++++++++++++++++++ environment.yml | 2 +- include/xtensor/xtensor_config.hpp | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 10fe9f240..4ede9d9cd 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| | master | ^0.6.18 | ^7.4.8 | +| 0.21.7 | ^0.6.18 | ^7.4.8 | | 0.21.6 | ^0.6.18 | ^7.4.8 | | 0.21.5 | ^0.6.12 | ^7.4.6 | | 0.21.4 | ^0.6.12 | ^7.4.6 | diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 46d9edd0d..88af028f8 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -7,6 +7,24 @@ Changelog ========= +0.21.7 +------ + +- Removed zheaders from single header + `#2157 `_ +- Implemented insertion of range and intializer list in svector + `#2165 `_ +- Adding has_shape + `#2163 `_ +- Adding get_rank and has_fixed_rank + `#2162 `_ +- Zrefactoring + `#2140 `_ +- Added missing header + `#2169 `_ +- Extending docs random + `#2173 `_ + 0.21.6 ------ diff --git a/environment.yml b/environment.yml index d3c2bcebc..9cf73c357 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xtensor channels: - conda-forge dependencies: - - xtensor=0.21.6 + - xtensor=0.21.7 - xtensor-blas=0.17.2 - xeus-cling=0.8.1 - blas * *openblas" diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index f7e75f9cc..46206aaf9 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -12,7 +12,7 @@ #define XTENSOR_VERSION_MAJOR 0 #define XTENSOR_VERSION_MINOR 21 -#define XTENSOR_VERSION_PATCH 6 +#define XTENSOR_VERSION_PATCH 7 // DETECT 3.6 <= clang < 3.8 for compiler bug workaround. #ifdef __clang__ From 452a47df9e7aa8b49c6d540c55139f97fc019a90 Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Mon, 5 Oct 2020 11:43:28 +0200 Subject: [PATCH 108/606] Fix undefined behavior while testing shifts shifting by more than register width is undefined behavior, not the perfect test :-) Fix #2174 --- test/test_xoperation.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test_xoperation.cpp b/test/test_xoperation.cpp index f9748c097..20a083a40 100644 --- a/test/test_xoperation.cpp +++ b/test/test_xoperation.cpp @@ -807,17 +807,18 @@ namespace xt TEST(operation, left_shift) { xarray arr({5,1, 1000}); + xarray arr2({2,1, 3}); xarray res1 = left_shift(arr, 4); - xarray res2 = left_shift(arr, arr); + xarray res2 = left_shift(arr, arr2); EXPECT_EQ(left_shift(arr, 4)(1), 16); xarray expected1 = {80, 16, 16000}; - xarray expected2 = {160, 2, 256000}; + xarray expected2 = {20, 2, 8000}; EXPECT_EQ(expected1, res1); EXPECT_EQ(expected2, res2); xarray res3 = arr << 4; - xarray res4 = arr << arr; + xarray res4 = arr << arr2; EXPECT_EQ(expected1, res3); EXPECT_EQ(expected2, res4); } @@ -825,17 +826,18 @@ namespace xt TEST(operation, right_shift) { xarray arr({5,1, 1000}); + xarray arr2({2,1, 3}); xarray res1 = right_shift(arr, 4); - xarray res2 = right_shift(arr, arr); + xarray res2 = right_shift(arr, arr2); EXPECT_EQ(right_shift(arr, 4)(1), 0); xarray expected1 = {0, 0, 62}; - xarray expected2 = {0, 0, 3}; + xarray expected2 = {1, 0, 125}; EXPECT_EQ(expected1, res1); EXPECT_EQ(expected2, res2); xarray res3 = arr >> 4; - xarray res4 = arr >> arr; + xarray res4 = arr >> arr2; EXPECT_EQ(expected1, res3); EXPECT_EQ(expected2, res4); } From 23d07754d000129f153d3c1c7abd095c3c525813 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 5 Oct 2020 15:45:09 +0200 Subject: [PATCH 109/606] Fix xchunked_array assignment --- include/xtensor/xchunk_store_manager.hpp | 25 ++++++++- include/xtensor/xchunked_array.hpp | 71 +++++++++++++++--------- include/xtensor/xfile_array.hpp | 51 ++++++++++------- test/test_xchunked_array.cpp | 22 ++++---- 4 files changed, 113 insertions(+), 56 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 667bd18d2..597a8ac33 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -108,6 +108,8 @@ namespace xt template void resize(S&& shape); + std::size_t size(); + void set_pool_size(std::size_t n); void set_directory(const char* directory); IP& get_index_path(); @@ -119,6 +121,9 @@ namespace xt template reference map_file_array(I first, I last); + template + const_reference map_file_array(I first, I last) const; + private: template @@ -128,6 +133,7 @@ namespace xt using index_pool_type = std::vector; shape_type m_shape; + shape_type m_shape_internal; chunk_pool_type m_chunk_pool; index_pool_type m_index_pool; std::size_t m_unload_index; @@ -169,6 +175,7 @@ namespace xt template inline xchunk_store_manager::xchunk_store_manager() : m_shape() + , m_shape_internal() // default pool size is 1 // so that first chunk is always resized to the chunk shape , m_chunk_pool(1u) @@ -248,10 +255,17 @@ namespace xt template template - inline void xchunk_store_manager::resize(S&&) + inline void xchunk_store_manager::resize(S&& shape) { // don't resize according to total number of chunks // instead the pool manages a number of in-memory chunks + m_shape_internal = shape; + } + + template + inline std::size_t xchunk_store_manager::size() + { + return compute_size(m_shape); } template @@ -300,6 +314,8 @@ namespace xt void xchunk_store_manager::set_directory(const char* directory) { m_index_path.set_directory(directory); + // make shape public + m_shape = m_shape_internal; } template @@ -345,6 +361,13 @@ namespace xt } } + template + template + inline auto xchunk_store_manager::map_file_array(I first, I last) const -> const_reference + { + return const_cast*>(this)->map_file_array(first, last); + } + template template inline std::array diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 73c09cb81..74c3ab722 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -4,7 +4,6 @@ #include #include -#include "xarray.hpp" #include "xnoalias.hpp" #include "xstrided_view.hpp" @@ -68,6 +67,7 @@ namespace xt using temporary_type = typename inner_types::temporary_type; using bool_load_type = xt::bool_load_type; static constexpr layout_type static_layout = layout_type::dynamic; + static constexpr bool contiguous_layout = false; template xchunked_array(S&& shape, S&& chunk_shape); @@ -88,6 +88,9 @@ namespace xt template xchunked_array& operator=(const xexpression& e); + template + void assign(const xexpression& e); + const shape_type& shape() const noexcept; layout_type layout() const noexcept; bool is_contiguous() const noexcept; @@ -107,6 +110,9 @@ namespace xt template bool broadcast_shape(S& s, bool reuse_cache = false) const; + template + bool has_linear_assign(const S& strides) const noexcept; + template stepper stepper_begin(const S& shape) noexcept; template @@ -124,15 +130,15 @@ namespace xt private: template - using indexes_type = std::pair, std::array>; + using indexes_type = std::pair, std::array>; template - using chunk_indexes_type = std::array, sizeof...(Idxs)>; + using chunk_indexes_type = std::array, sizeof...(Idxs)>; template - using static_indexes_type = std::pair, std::array>; + using static_indexes_type = std::pair, std::array>; - using dynamic_indexes_type = std::pair, std::vector>; + using dynamic_indexes_type = std::pair, std::vector>; template void resize(S1&& shape, S2&& chunk_shape); @@ -141,9 +147,9 @@ namespace xt indexes_type get_indexes(Idxs... idxs) const; template - std::pair get_chunk_indexes_in_dimension(size_t dim, Idx idx) const; + std::pair get_chunk_indexes_in_dimension(std::size_t dim, Idx idx) const; - template + template chunk_indexes_type get_chunk_indexes(std::index_sequence, Idxs... idxs) const; template @@ -224,10 +230,16 @@ namespace xt inline xchunked_array::xchunked_array(const xexpression& e, S&& chunk_shape) { resize(e.derived_cast().shape(), std::forward(chunk_shape)); + assign(e); + } + + template + template + inline void xchunked_array::assign(const xexpression& e) + { xstrided_slice_vector sv(m_chunk_shape.size()); // element slice corresponding to chunk std::transform(m_chunk_shape.begin(), m_chunk_shape.end(), sv.begin(), [](auto size) { return range(0, size); }); - shape_type ic(this->dimension()); // index of chunk, initialized to 0... size_type ci = 0; for (auto& chunk: m_chunks) @@ -251,7 +263,6 @@ namespace xt { di--; } - } else { @@ -269,7 +280,8 @@ namespace xt template inline auto xchunked_array::operator=(const xexpression& e) -> self_type& { - return semantic_base::operator=(e); + assign(e); + return *this; } template @@ -279,7 +291,7 @@ namespace xt } template - inline auto xchunked_array::layout() const noexcept -> layout_type + inline auto xchunked_array::layout() const noexcept -> layout_type { return static_layout; } @@ -333,6 +345,13 @@ namespace xt return xt::broadcast_shape(shape(), s); } + template + template + inline bool xchunked_array::has_linear_assign(const S& strides) const noexcept + { + return false; + } + template template inline auto xchunked_array::stepper_begin(const S& shape) noexcept -> stepper @@ -388,7 +407,7 @@ namespace xt inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) { // compute chunk number in each dimension (shape_of_chunks) - std::vector shape_of_chunks(shape.size()); + std::vector shape_of_chunks(shape.size()); std::transform ( shape.cbegin(), shape.cend(), @@ -396,14 +415,14 @@ namespace xt shape_of_chunks.begin(), [](auto s, auto cs) { - size_t cn = s / cs; + std::size_t cn = s / cs; if (s % cs > 0) - cn += size_t(1); // edge_chunk + cn += std::size_t(1); // edge_chunk return cn; } ); - // resize the xarray of chunks + // resize the chunk container m_chunks.resize(shape_of_chunks); // resize each chunk for (auto& c: m_chunks) @@ -425,15 +444,15 @@ namespace xt template template - inline std::pair xchunked_array::get_chunk_indexes_in_dimension(size_t dim, Idx idx) const + inline std::pair xchunked_array::get_chunk_indexes_in_dimension(std::size_t dim, Idx idx) const { - size_t index_of_chunk = static_cast(idx) / m_chunk_shape[dim]; - size_t index_in_chunk = static_cast(idx) - index_of_chunk * m_chunk_shape[dim]; + std::size_t index_of_chunk = static_cast(idx) / m_chunk_shape[dim]; + std::size_t index_in_chunk = static_cast(idx) - index_of_chunk * m_chunk_shape[dim]; return std::make_pair(index_of_chunk, index_in_chunk); } template - template + template inline auto xchunked_array::get_chunk_indexes(std::index_sequence, Idxs... idxs) const -> chunk_indexes_type { @@ -445,9 +464,9 @@ namespace xt template inline auto xchunked_array::unpack(const std::array &arr) const -> static_indexes_type { - std::array arr0; - std::array arr1; - for (size_t i = 0; i < N; ++i) + std::array arr0; + std::array arr1; + for (std::size_t i = 0; i < N; ++i) { arr0[i] = std::get<0>(arr[i]); arr1[i] = std::get<1>(arr[i]); @@ -459,10 +478,10 @@ namespace xt template inline auto xchunked_array::get_indexes_dynamic(It first, It last) const -> dynamic_indexes_type { - auto size = static_cast(std::distance(first, last)); - std::vector indexes_of_chunk(size); - std::vector indexes_in_chunk(size); - for (size_t dim = 0; dim < size; ++dim) + auto size = static_cast(std::distance(first, last)); + std::vector indexes_of_chunk(size); + std::vector indexes_in_chunk(size); + for (std::size_t dim = 0; dim < size; ++dim) { auto chunk_index = get_chunk_indexes_in_dimension(dim, *first++); indexes_of_chunk[dim] = chunk_index.first; diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index ee7cde766..666384061 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -12,18 +12,18 @@ namespace xt { template - class xfile_reference + class xfile_value_reference { public: - using self_type = xfile_reference; + using self_type = xfile_value_reference; using const_reference = const T&; - xfile_reference(T& value, bool& dirty); - ~xfile_reference() = default; + xfile_value_reference(T& value, bool& dirty); + ~xfile_value_reference() = default; - xfile_reference(const xfile_reference&) = default; - xfile_reference(xfile_reference&&) = default; + xfile_value_reference(const xfile_value_reference&) = default; + xfile_value_reference(xfile_value_reference&&) = default; self_type& operator=(const self_type&); self_type& operator=(self_type&&); @@ -50,7 +50,19 @@ namespace xt T& m_value; bool& m_dirty; }; +} +namespace std +{ + template + struct is_signed> + : is_signed + { + }; +} + +namespace xt +{ template class xfile_array_container; @@ -59,7 +71,7 @@ namespace xt { using storage_type = E; using value_type = typename storage_type::value_type; - using reference = xfile_reference; + using reference = xfile_value_reference; using const_reference = typename storage_type::const_reference; using size_type = typename storage_type::size_type; using temporary_type = xfile_array_container; @@ -99,6 +111,7 @@ namespace xt using temporary_type = typename inner_types::temporary_type; using bool_load_type = xt::bool_load_type; static constexpr layout_type static_layout = layout_type::dynamic; + static constexpr bool contiguous_layout = true; xfile_array_container() = default; ~xfile_array_container(); @@ -182,7 +195,7 @@ namespace xt const std::string& path() const noexcept; void ignore_empty_path(bool ignore); - void set_path(std::string& path); + void set_path(const std::string& path); template void configure_format(C& config); @@ -207,19 +220,19 @@ namespace xt class SA = std::allocator::size_type>> using xfile_array = xfile_array_container, IOH>; - /********************************** - * xfile_reference implementation * - **********************************/ + /**************************************** + * xfile_value_reference implementation * + ****************************************/ template - inline xfile_reference::xfile_reference(T& value, bool& dirty) + inline xfile_value_reference::xfile_value_reference(T& value, bool& dirty) : m_value(value), m_dirty(dirty) { } template template - inline auto xfile_reference::operator=(const V& v) -> self_type& + inline auto xfile_value_reference::operator=(const V& v) -> self_type& { if (v != m_value) { @@ -231,7 +244,7 @@ namespace xt template template - inline auto xfile_reference::operator+=(const V& v) -> self_type& + inline auto xfile_value_reference::operator+=(const V& v) -> self_type& { if (v != T(0)) { @@ -243,7 +256,7 @@ namespace xt template template - inline auto xfile_reference::operator-=(const V& v) -> self_type& + inline auto xfile_value_reference::operator-=(const V& v) -> self_type& { if (v != T(0)) { @@ -255,7 +268,7 @@ namespace xt template template - inline auto xfile_reference::operator*=(const V& v) -> self_type& + inline auto xfile_value_reference::operator*=(const V& v) -> self_type& { if (v != T(1)) { @@ -267,7 +280,7 @@ namespace xt template template - inline auto xfile_reference::operator/=(const V& v) -> self_type& + inline auto xfile_value_reference::operator/=(const V& v) -> self_type& { if (v != T(1)) { @@ -278,7 +291,7 @@ namespace xt } template - inline xfile_reference::operator const_reference() const + inline xfile_value_reference::operator const_reference() const { return m_value; } @@ -567,7 +580,7 @@ namespace xt } template - inline void xfile_array_container::set_path(std::string& path) + inline void xfile_array_container::set_path(const std::string& path) { if (path != m_path) { diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 50fa8c08b..990f69b3c 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -18,13 +18,13 @@ namespace xt { - using chunked_array = xchunked_array>>; + using in_memory_chunked_array = xchunked_array>>; TEST(xchunked_array, indexed_access) { std::vector shape = {10, 10, 10}; std::vector chunk_shape = {2, 3, 4}; - chunked_array a(shape, chunk_shape); + in_memory_chunked_array a(shape, chunk_shape); std::vector idx = {3, 9, 8}; double val; @@ -53,7 +53,7 @@ namespace xt #endif std::vector shape1 = {2, 2, 2}; std::vector chunk_shape1 = {2, 3, 4}; - chunked_array a1(shape1, chunk_shape1); + in_memory_chunked_array a1(shape1, chunk_shape1); double val; val = 3.; @@ -64,7 +64,7 @@ namespace xt } std::vector shape2 = {32, 10, 10}; - chunked_array a2(shape2, chunk_shape1); + in_memory_chunked_array a2(shape2, chunk_shape1); a2 = broadcast(val, a2.shape()); for (const auto& v: a2) @@ -86,7 +86,7 @@ namespace xt EXPECT_EQ(is_chunked(a3), false); std::vector chunk_shape4 = {2, 2}; - auto a4 = chunked_array(a3, chunk_shape4); + auto a4 = in_memory_chunked_array(a3, chunk_shape4); EXPECT_EQ(is_chunked(a4), true); @@ -97,14 +97,14 @@ namespace xt i += 1.; } - auto a5 = chunked_array(a4); + auto a5 = in_memory_chunked_array(a4); EXPECT_EQ(is_chunked(a5), true); for (const auto& v: a5.chunk_shape()) { EXPECT_EQ(v, 2); } - auto a6 = chunked_array(a3); + auto a6 = in_memory_chunked_array(a3); EXPECT_EQ(is_chunked(a6), true); for (const auto& v: a6.chunk_shape()) { @@ -116,7 +116,9 @@ namespace xt { std::vector shape = {4, 4}; std::vector chunk_shape = {2, 2}; + std::string chunk_dir = "files"; xchunked_array>>> a1(shape, chunk_shape); + a1.chunks().set_directory(chunk_dir.c_str()); a1.chunks().set_pool_size(2); std::vector idx = {1, 2}; double v1 = 3.4; @@ -132,20 +134,20 @@ namespace xt std::ifstream in_file; xt::xarray ref; xt::xarray data; - in_file.open("1.0"); + in_file.open(chunk_dir + "/1.0"); data = xt::load_csv(in_file); ref = {{0, v1}, {0, 0}}; EXPECT_EQ(data, ref); in_file.close(); a1.chunks().flush(); - in_file.open("0.1"); + in_file.open(chunk_dir + "/0.1"); data = xt::load_csv(in_file); ref = {{0, 0}, {v2, 0}}; EXPECT_EQ(data, ref); in_file.close(); - in_file.open("0.0"); + in_file.open(chunk_dir + "/0.0"); data = xt::load_csv(in_file); ref = {{v3, 0}, {0, 0}}; EXPECT_EQ(data, ref); From fcee1925175252eb71d486fe1da01a64feede9ea Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 6 Oct 2020 17:11:29 +0200 Subject: [PATCH 110/606] Fix zarray initialization from zarray (#2180) Fix zarray initialization from zarray --- include/xtensor/zarray.hpp | 4 ++-- test/test_xfunction.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 8e299bc7b..653d6b23f 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -44,7 +44,7 @@ namespace xt zarray() = default; ~zarray() = default; - template + template , zarray>::value>> zarray(E&& e); zarray(implementation_ptr&& impl); @@ -98,7 +98,7 @@ namespace xt semantic_base::assign(e); } - template + template inline zarray::zarray(E&& e) { init_implementation(std::forward(e), extension::get_expression_tag_t>()); diff --git a/test/test_xfunction.cpp b/test/test_xfunction.cpp index a8ab538d8..ddd9ce824 100644 --- a/test/test_xfunction.cpp +++ b/test/test_xfunction.cpp @@ -465,6 +465,7 @@ namespace xt #endif } +#ifndef _MSC_VER TEST(xfunction, xfunction_in_xfunction) { using Point3 = xt::xtensor_fixed>; @@ -477,4 +478,5 @@ namespace xt xtensor res{r1, r2, r3}; EXPECT_EQ(c, res); } +#endif } From 20c18a47e05feb2fee27a24aaa914a86a48e11c8 Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Wed, 7 Oct 2020 07:03:11 +0200 Subject: [PATCH 111/606] Fix xnpy save padding computation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to the documentation [0], > It is terminated by a newline (\n) and padded with spaces (\x20) to make the > total of len(magic string) + 2 + len(length) + HEADER_LEN be evenly > divisible by 64 for alignment purposes. In former version, the aligment was 16 [1] > the total length of the magic string + 4 + HEADER_LEN be evenly divisible by > 16 for alignment purposes. The official documentation also states > The .npy format, including motivation for creating it and a comparison of > alternatives, is described in the “npy-format” NEP, however details have > evolved with time and this document is more current. This patches fixes xnpy implementation and reference tests to match the documentation. [0] https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html [1] https://docs.scipy.org/doc/numpy-1.14.2/neps/npy-format.html --- include/xtensor/xnpy.hpp | 2 +- test/files/xnpy_files/bool.npy | Bin 107 -> 155 bytes test/files/xnpy_files/bool_fortran.npy | Bin 107 -> 155 bytes test/files/xnpy_files/double.npy | Bin 296 -> 344 bytes test/files/xnpy_files/double_fortran.npy | Bin 296 -> 344 bytes test/files/xnpy_files/unsignedlong.npy | Bin 120 -> 168 bytes test/files/xnpy_files/unsignedlong_fortran.npy | Bin 120 -> 168 bytes 7 files changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xnpy.hpp b/include/xtensor/xnpy.hpp index 9f86c03a6..12deca9b0 100644 --- a/include/xtensor/xnpy.hpp +++ b/include/xtensor/xnpy.hpp @@ -393,7 +393,7 @@ namespace xt version[0] = 2; version[1] = 0; } - std::size_t padding_len = 16 - metadata_len % 16; + std::size_t padding_len = 64 - (metadata_len % 64); std::string padding(padding_len, ' '); ss_header << padding; ss_header << std::endl; diff --git a/test/files/xnpy_files/bool.npy b/test/files/xnpy_files/bool.npy index 3e4cd6e1c22a33d36f23bde4a06a264376219f43..528c0eefb7a173b90a0249a82500bb061f2dd8e5 100644 GIT binary patch delta 61 ecmd0v&B!^~FVr_6l98coBBz_C0%y9xktxeB-d delta 19 acmbQun9VuaFVr_6l99n}BB$HLdKmyXbOn$A diff --git a/test/files/xnpy_files/double.npy b/test/files/xnpy_files/double.npy index e5ef170135d788c9153e27f40b314d63e19b746f..b4eb19a0d7c069c53cd046d0da40bfcad66b5497 100644 GIT binary patch delta 62 fcmZ3%bc2a=vR|lgKqMnW*+fn^O$B1XM2`mmn-dHZ delta 20 bcmcb?w1SCqvR|lgKqMoB+eA*ciS;)DLwp9x diff --git a/test/files/xnpy_files/double_fortran.npy b/test/files/xnpy_files/double_fortran.npy index c6a0bb7764813b2dadf080e8cbcc459754cd3ab6..0d395e90bbfaafb8d01fdd60c0587fc90fdeab29 100644 GIT binary patch delta 61 fcmZ3%bc2a=vR|lgKqMnW*+fn^4Fw{>ME8dPkSq)G delta 20 bcmcb?w1SCqvR|lgKqMoB+eA*ciS;)DLwp9x diff --git a/test/files/xnpy_files/unsignedlong.npy b/test/files/xnpy_files/unsignedlong.npy index f4636857438af347e46db797b99f4351e2d2a4cc..4c6863fb98fa6d50a4dead2aa48a86b7e6bbed75 100644 GIT binary patch delta 56 ecmbi6$)Db delta 19 acmZ3%Siw2jFVr_6l99n}BB$HLdQAX2RRy;I diff --git a/test/files/xnpy_files/unsignedlong_fortran.npy b/test/files/xnpy_files/unsignedlong_fortran.npy index f4636857438af347e46db797b99f4351e2d2a4cc..4c6863fb98fa6d50a4dead2aa48a86b7e6bbed75 100644 GIT binary patch delta 56 ecmbi6$)Db delta 19 acmZ3%Siw2jFVr_6l99n}BB$HLdQAX2RRy;I From 808ceb4a803909a309a00d72fbfd633d81843ea8 Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Tue, 6 Oct 2020 20:54:29 +0200 Subject: [PATCH 112/606] Portable and generic implementation of endianess detection No need to have it non-portable just for the sake of being constexpr, especially as both gcc and clang fold the result at -O2. --- include/xtensor/xnpy.hpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/include/xtensor/xnpy.hpp b/include/xtensor/xnpy.hpp index 9f86c03a6..2e7882023 100644 --- a/include/xtensor/xnpy.hpp +++ b/include/xtensor/xnpy.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -42,17 +43,15 @@ namespace xt namespace detail { - /* Compile-time test for byte order. - If your compiler does not define these per default, you may want to define - one of these constants manually. - Defaults to little endian order. */ -#if defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN || defined(__BIG_ENDIAN__) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(_MIBSEB) || defined(__MIBSEB) || defined(__MIBSEB__) - const bool big_endian = true; -#else - const bool big_endian = false; -#endif + /* Test for endianess. Compiler can optimize that to a single constant. */ + static inline bool is_big_endian() + { + uint32_t utmp = 0x01020304; + char btmp[sizeof(utmp)]; + std::memcpy(&btmp[0], &utmp, sizeof(utmp)); + const bool big_endian = btmp[0] == 0x01; + return big_endian; + } const char magic_string[] = "\x93NUMPY"; const std::size_t magic_string_length = 6; @@ -61,7 +60,7 @@ namespace xt const char big_endian_char = '>'; const char no_endian_char = '|'; - constexpr char host_endian_char = (big_endian ? big_endian_char : little_endian_char); + char host_endian_char = (is_big_endian() ? big_endian_char : little_endian_char); template inline void write_magic(O& ostream, From cf840966b27d18200cd423cf8ecb923cb0179e83 Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Wed, 7 Oct 2020 10:02:30 +0200 Subject: [PATCH 113/606] Only use -march=native if it's available Not all architecture support this flag, e.g. gcc on ppc64le doesn't support this flag. --- benchmark/CMakeLists.txt | 3 ++- docs/source/build-options.rst | 3 ++- docs/source/getting_started.rst | 2 +- test/CMakeLists.txt | 6 ++++-- xtensorConfig.cmake.in | 5 ++++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 267dff5c1..4dc77a93d 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -23,7 +23,8 @@ include(CheckCXXCompilerFlag) string(TOUPPER "${CMAKE_BUILD_TYPE}" U_CMAKE_BUILD_TYPE) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Intel") - if(NOT CMAKE_CXX_FLAGS MATCHES "-march") + CHECK_CXX_COMPILER_FLAG(-march=native arch_native_supported) + if(arch_native_supported AND NOT CMAKE_CXX_FLAGS MATCHES "-march") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -Wunused-parameter -Wextra -Wreorder") diff --git a/docs/source/build-options.rst b/docs/source/build-options.rst index b712dd6ea..8f91fd3b3 100644 --- a/docs/source/build-options.rst +++ b/docs/source/build-options.rst @@ -103,7 +103,8 @@ anything, the system will do its best to enable the most recent supported instru Linux/OSX ~~~~~~~~~ -Whether you enabled ``XTENSOR_USE_XSIMD`` or not, it is highly recommended to build with ``-march=native`` option: +Whether you enabled ``XTENSOR_USE_XSIMD`` or not, it is highly recommended to build with ``-march=native`` option, +if your compiler supports it: .. code:: cmake diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 4f675a478..134c08dac 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -99,7 +99,7 @@ The following minimal ``CMakeLists.txt`` is enough to build the first example: target_link_libraries(... xtensor::optimize) - set the following compiler flags: + set the following compiler flags, if supported by the target compiler: * Unix: ``-march=native``; * Windows: ``/EHsc /MP /bigobj``. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9c15c1ea2..ebb72b0a0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -63,7 +63,8 @@ if(NOT _cxx_std_flag) endif() if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR (CMAKE_CXX_COMPILER_ID MATCHES "Intel" AND NOT WIN32)) - if(NOT CMAKE_CXX_FLAGS MATCHES "-march") + CHECK_CXX_COMPILER_FLAG(-march=native arch_native_supported) + if(arch_native_supported AND NOT CMAKE_CXX_FLAGS MATCHES "-march") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_cxx_std_flag} -Wunused-parameter -Wextra -Wreorder -Wconversion -Wsign-conversion") @@ -83,7 +84,8 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(NOT WIN32) - if(NOT CMAKE_CXX_FLAGS MATCHES "-march") + CHECK_CXX_COMPILER_FLAG(-march=native arch_native_supported) + if(arch_native_supported AND NOT CMAKE_CXX_FLAGS MATCHES "-march") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_cxx_std_flag} -Wunused-parameter -Wextra -Wreorder -Wconversion -Wsign-conversion") diff --git a/xtensorConfig.cmake.in b/xtensorConfig.cmake.in index 37391dab9..24989fa5d 100644 --- a/xtensorConfig.cmake.in +++ b/xtensorConfig.cmake.in @@ -43,7 +43,10 @@ if (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} VERSION_GREATER_EQUAL 3.11) target_compile_options(xtensor::optimize INTERFACE /EHsc /MP /bigobj) # gcc, clang, ... else() - target_compile_options(xtensor::optimize INTERFACE -march=native) + CHECK_CXX_COMPILER_FLAG(-march=native arch_native_supported) + if(arch_native_supported) + target_compile_options(xtensor::optimize INTERFACE -march=native) + endif() endif() endif() From b0e09616ade17c91efebd586897dc75c582f224b Mon Sep 17 00:00:00 2001 From: David Brochart Date: Wed, 7 Oct 2020 16:35:49 +0200 Subject: [PATCH 114/606] Add specific xchunked_array constructor for xchunk_store_manager --- include/xtensor/xchunk_store_manager.hpp | 14 ++++-------- include/xtensor/xchunked_array.hpp | 29 ++++++++++++++++++++++-- test/test_xchunked_array.cpp | 5 ++-- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 597a8ac33..5a1aee37a 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -18,7 +18,7 @@ namespace xt class xindex_path { public: - void set_directory(const char* directory); + void set_directory(const std::string& directory); template void index_to_path(I, I, std::string&); @@ -111,7 +111,7 @@ namespace xt std::size_t size(); void set_pool_size(std::size_t n); - void set_directory(const char* directory); + void set_directory(const std::string& directory); IP& get_index_path(); void flush(); @@ -133,7 +133,6 @@ namespace xt using index_pool_type = std::vector; shape_type m_shape; - shape_type m_shape_internal; chunk_pool_type m_chunk_pool; index_pool_type m_index_pool; std::size_t m_unload_index; @@ -144,7 +143,7 @@ namespace xt * xindex_path implementation * ******************************/ - void xindex_path::set_directory(const char* directory) + void xindex_path::set_directory(const std::string& directory) { m_directory = directory; if (m_directory.back() != '/') @@ -175,7 +174,6 @@ namespace xt template inline xchunk_store_manager::xchunk_store_manager() : m_shape() - , m_shape_internal() // default pool size is 1 // so that first chunk is always resized to the chunk shape , m_chunk_pool(1u) @@ -259,7 +257,7 @@ namespace xt { // don't resize according to total number of chunks // instead the pool manages a number of in-memory chunks - m_shape_internal = shape; + m_shape = shape; } template @@ -311,11 +309,9 @@ namespace xt } template - void xchunk_store_manager::set_directory(const char* directory) + void xchunk_store_manager::set_directory(const std::string& directory) { m_index_path.set_directory(directory); - // make shape public - m_shape = m_shape_internal; } template diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 74c3ab722..d741bbfee 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -6,10 +6,22 @@ #include "xnoalias.hpp" #include "xstrided_view.hpp" +#include "xchunk_store_manager.hpp" namespace xt { + template + struct is_chunk_store_manager : std::false_type + { + }; + + template + struct is_chunk_store_manager> + : std::true_type + { + }; + /****************************** * xchunked_array declaration * ******************************/ @@ -69,8 +81,12 @@ namespace xt static constexpr layout_type static_layout = layout_type::dynamic; static constexpr bool contiguous_layout = false; - template + template ::value>> + xchunked_array(S&& shape, S&& chunk_shape, const std::string& directory, std::size_t pool_size = 1); + + template ::value>> xchunked_array(S&& shape, S&& chunk_shape); + ~xchunked_array() = default; xchunked_array(const xchunked_array&) = default; @@ -212,7 +228,16 @@ namespace xt *********************************/ template - template + template + inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape, const std::string& directory, std::size_t pool_size) + { + resize(std::forward(shape), std::forward(chunk_shape)); + m_chunks.set_directory(directory); + m_chunks.set_pool_size(pool_size); + } + + template + template inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape) { resize(std::forward(shape), std::forward(chunk_shape)); diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 990f69b3c..85ee4c781 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -117,9 +117,8 @@ namespace xt std::vector shape = {4, 4}; std::vector chunk_shape = {2, 2}; std::string chunk_dir = "files"; - xchunked_array>>> a1(shape, chunk_shape); - a1.chunks().set_directory(chunk_dir.c_str()); - a1.chunks().set_pool_size(2); + std::size_t pool_size = 2; + xchunked_array>>> a1(shape, chunk_shape, chunk_dir, pool_size); std::vector idx = {1, 2}; double v1 = 3.4; double v2 = 5.6; From d1de15ca3e899b0679dab4ff52e7fc4330b6d93d Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Wed, 7 Oct 2020 18:53:24 +0200 Subject: [PATCH 115/606] Make xnpy tests aware of both little and big endian targets Provide reference files for both endianess system, and pick the right one at runtime. Fix #2181 --- test/CMakeLists.txt | 21 +++++---- .../xnpy_files/{bool.npy => bool.be.npy} | Bin test/files/xnpy_files/bool.le.npy | Bin 0 -> 155 bytes .../{bool_fortran.npy => bool_fortran.be.npy} | Bin test/files/xnpy_files/bool_fortran.le.npy | Bin 0 -> 155 bytes .../xnpy_files/{double.npy => double.be.npy} | Bin test/files/xnpy_files/double.le.npy | Bin 0 -> 344 bytes ...uble_fortran.npy => double_fortran.be.npy} | Bin test/files/xnpy_files/double_fortran.le.npy | Bin 0 -> 344 bytes test/files/xnpy_files/{int.npy => int.be.npy} | Bin test/files/xnpy_files/int.le.npy | Bin 0 -> 148 bytes .../{unsignedlong.npy => unsignedlong.be.npy} | Bin ...edlong_fortran.npy => unsignedlong.le.npy} | Bin .../xnpy_files/unsignedlong_fortran.be.npy | Bin 0 -> 168 bytes .../xnpy_files/unsignedlong_fortran.le.npy | Bin 0 -> 168 bytes test/test_xnpy.cpp | 40 +++++++++--------- 16 files changed, 32 insertions(+), 29 deletions(-) rename test/files/xnpy_files/{bool.npy => bool.be.npy} (100%) create mode 100644 test/files/xnpy_files/bool.le.npy rename test/files/xnpy_files/{bool_fortran.npy => bool_fortran.be.npy} (100%) create mode 100644 test/files/xnpy_files/bool_fortran.le.npy rename test/files/xnpy_files/{double.npy => double.be.npy} (100%) create mode 100644 test/files/xnpy_files/double.le.npy rename test/files/xnpy_files/{double_fortran.npy => double_fortran.be.npy} (100%) create mode 100644 test/files/xnpy_files/double_fortran.le.npy rename test/files/xnpy_files/{int.npy => int.be.npy} (100%) create mode 100644 test/files/xnpy_files/int.le.npy rename test/files/xnpy_files/{unsignedlong.npy => unsignedlong.be.npy} (100%) rename test/files/xnpy_files/{unsignedlong_fortran.npy => unsignedlong.le.npy} (100%) create mode 100644 test/files/xnpy_files/unsignedlong_fortran.be.npy create mode 100644 test/files/xnpy_files/unsignedlong_fortran.le.npy diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9c15c1ea2..dca304a2a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -240,17 +240,20 @@ endif() # Add files for npy tests set(XNPY_FILES - bool.npy - bool_fortran.npy - double.npy - double_fortran.npy - int.npy - unsignedlong.npy - unsignedlong_fortran.npy + bool + bool_fortran + double + double_fortran + int + unsignedlong + unsignedlong_fortran ) + foreach(filename IN LISTS XNPY_FILES) - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/files/xnpy_files/${filename} - ${CMAKE_CURRENT_BINARY_DIR}/files/xnpy_files/${filename} COPYONLY) + foreach(suffix .be.npy .le.npy) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/files/xnpy_files/${filename}${suffix} + ${CMAKE_CURRENT_BINARY_DIR}/files/xnpy_files/${filename}${suffix} COPYONLY) + endforeach() endforeach() set(XTENSOR_PREPROCESS_FILES diff --git a/test/files/xnpy_files/bool.npy b/test/files/xnpy_files/bool.be.npy similarity index 100% rename from test/files/xnpy_files/bool.npy rename to test/files/xnpy_files/bool.be.npy diff --git a/test/files/xnpy_files/bool.le.npy b/test/files/xnpy_files/bool.le.npy new file mode 100644 index 0000000000000000000000000000000000000000..528c0eefb7a173b90a0249a82500bb061f2dd8e5 GIT binary patch literal 155 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1JlVqr_qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= nXCxM+0{I%oItn19siRP#0F6SV{9AJ literal 0 HcmV?d00001 diff --git a/test/files/xnpy_files/bool_fortran.npy b/test/files/xnpy_files/bool_fortran.be.npy similarity index 100% rename from test/files/xnpy_files/bool_fortran.npy rename to test/files/xnpy_files/bool_fortran.be.npy diff --git a/test/files/xnpy_files/bool_fortran.le.npy b/test/files/xnpy_files/bool_fortran.le.npy new file mode 100644 index 0000000000000000000000000000000000000000..a6766f6a40ea0491e363d3d4a87b9bd52b723022 GIT binary patch literal 155 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1JlVqr_qoAIaUsO_*m=~X4l#&V(4=E~51qv5u kBo?Fsxf;eg3LvDZqfo0rCg5UV1VKgy1~3KY09jxW08~mFrvLx| literal 0 HcmV?d00001 diff --git a/test/files/xnpy_files/double.npy b/test/files/xnpy_files/double.be.npy similarity index 100% rename from test/files/xnpy_files/double.npy rename to test/files/xnpy_files/double.be.npy diff --git a/test/files/xnpy_files/double.le.npy b/test/files/xnpy_files/double.le.npy new file mode 100644 index 0000000000000000000000000000000000000000..b4eb19a0d7c069c53cd046d0da40bfcad66b5497 GIT binary patch literal 344 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+i=qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I%oItn19siRPKA*iL$E|6${+_9yvw>?f%&b7l{|VBdD-?U$W8@9h~5MKMe|aoC<=`_?}@gx1;j zw!V3qtM}C2fyX#TbWN}QlB$Z@=9XjjGOnLa)$BTGzi%6>N~GjF`-A6;Y_3juX}|W% sb@%tF5A0dird~gr|G>V**OI}j?1Ft+z^G$>+D;6Eg8JZF4!*!e6&L&cbWaM z(|i@SlOEg4xPCfSv+JOJrRasNai1R8w_SPrWv9-2`?X)LyT4C;U|**;HUCY)J^S9) zH&1i*p4yjHeq}aVc+1|v^%>WRz<2f^Hoj_ql7Gj3-!@j2NXd8h7mT0V#w=K8&u}P; rVbY1i_AF~tub<6-V1L%JY~ziZXZ8*}#xbI6dhJ(uNG8`XZ?FddZ?lMr literal 0 HcmV?d00001 diff --git a/test/files/xnpy_files/int.npy b/test/files/xnpy_files/int.be.npy similarity index 100% rename from test/files/xnpy_files/int.npy rename to test/files/xnpy_files/int.be.npy diff --git a/test/files/xnpy_files/int.le.npy b/test/files/xnpy_files/int.le.npy new file mode 100644 index 0000000000000000000000000000000000000000..d24dd3f56963a4d7d94e8335184a66bb8ca8f945 GIT binary patch literal 148 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= jXCxM+0{I%II+{8PwF(pfE@lP>1{NS@1!6WJW(Q&bc`6%t literal 0 HcmV?d00001 diff --git a/test/files/xnpy_files/unsignedlong.npy b/test/files/xnpy_files/unsignedlong.be.npy similarity index 100% rename from test/files/xnpy_files/unsignedlong.npy rename to test/files/xnpy_files/unsignedlong.be.npy diff --git a/test/files/xnpy_files/unsignedlong_fortran.npy b/test/files/xnpy_files/unsignedlong.le.npy similarity index 100% rename from test/files/xnpy_files/unsignedlong_fortran.npy rename to test/files/xnpy_files/unsignedlong.le.npy diff --git a/test/files/xnpy_files/unsignedlong_fortran.be.npy b/test/files/xnpy_files/unsignedlong_fortran.be.npy new file mode 100644 index 0000000000000000000000000000000000000000..4c6863fb98fa6d50a4dead2aa48a86b7e6bbed75 GIT binary patch literal 168 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZQ);2EqoAIaUsO_*m=~X4l#&V(cT3DEP6dh= lXCxM+0{I%II+{8PwF(pfE*=I5;DgcvP+AB|PrM3Y0RXkr9U}k$ literal 0 HcmV?d00001 diff --git a/test/files/xnpy_files/unsignedlong_fortran.le.npy b/test/files/xnpy_files/unsignedlong_fortran.le.npy new file mode 100644 index 0000000000000000000000000000000000000000..4c6863fb98fa6d50a4dead2aa48a86b7e6bbed75 GIT binary patch literal 168 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZQ);2EqoAIaUsO_*m=~X4l#&V(cT3DEP6dh= lXCxM+0{I%II+{8PwF(pfE*=I5;DgcvP+AB|PrM3Y0RXkr9U}k$ literal 0 HcmV?d00001 diff --git a/test/test_xnpy.cpp b/test/test_xnpy.cpp index 651c37743..49df9fb32 100644 --- a/test/test_xnpy.cpp +++ b/test/test_xnpy.cpp @@ -17,6 +17,13 @@ namespace xt { + std::string get_load_filename(std::string const& npy_prefix, layout_type lt = layout_type::row_major) { + using detail::is_big_endian; + std::string lts = lt == layout_type::row_major ? "" : "_fortran"; + std::string endianess = is_big_endian() ? ".be" : ".le"; + return npy_prefix + lts + endianess + ".npy"; + } + TEST(xnpy, load) { xarray darr = {{{ 0.29731723, 0.04380157, 0.94748308}, @@ -41,28 +48,28 @@ namespace xt xarray iarr1d = {3, 4, 5, 6, 7}; - auto darr_loaded = load_npy("files/xnpy_files/double.npy"); + auto darr_loaded = load_npy(get_load_filename("files/xnpy_files/double")); EXPECT_TRUE(all(isclose(darr, darr_loaded))); - std::ifstream dstream("files/xnpy_files/double.npy"); + std::ifstream dstream(get_load_filename("files/xnpy_files/double")); auto darr_loaded_stream = load_npy(dstream); EXPECT_TRUE(all(isclose(darr, darr_loaded_stream))) << "Loading double numpy array from stream failed"; dstream.close(); - auto barr_loaded = load_npy("files/xnpy_files/bool.npy"); + auto barr_loaded = load_npy(get_load_filename("files/xnpy_files/bool")); EXPECT_TRUE(all(equal(barr, barr_loaded))); - std::ifstream bstream("files/xnpy_files/bool.npy"); + std::ifstream bstream(get_load_filename("files/xnpy_files/bool")); auto barr_loaded_stream = load_npy(bstream); EXPECT_TRUE(all(equal(barr, barr_loaded_stream))) << "Loading boolean numpy array from stream failed"; bstream.close(); - auto dfarr_loaded = load_npy("files/xnpy_files/double_fortran.npy"); + auto dfarr_loaded = load_npy(get_load_filename("files/xnpy_files/double_fortran")); EXPECT_TRUE(all(isclose(darr, dfarr_loaded))); - auto iarr1d_loaded = load_npy("files/xnpy_files/int.npy"); + auto iarr1d_loaded = load_npy(get_load_filename("files/xnpy_files/int")); EXPECT_TRUE(all(equal(iarr1d, iarr1d_loaded))); } @@ -79,7 +86,8 @@ namespace xt fn1_contents.size() == fn2_contents.size(); } - std::string get_filename(int n) + + std::string get_dump_filename(int n) { std::string filename = "files/xnpy_files/test_dump_" + std::to_string(n) + ".npy"; return filename; @@ -92,7 +100,7 @@ namespace xt TEST(xnpy, dump) { - std::string filename = get_filename(0); + std::string filename = get_dump_filename(0); xarray barr = {{{0, 0, 1}, {1, 1, 0}, {1, 0, 1}}, @@ -106,11 +114,7 @@ namespace xt xtensor ularr = {12ul, 14ul, 16ul, 18ul, 1234321ul}; dump_npy(filename, barr); - std::string compare_name = "files/xnpy_files/bool.npy"; - if (barr.layout() == layout_type::column_major) - { - compare_name = "files/xnpy_files/bool_fortran.npy"; - } + std::string compare_name = get_load_filename("files/xnpy_files/bool", barr.layout()); EXPECT_TRUE(compare_binary_files(filename, compare_name)); @@ -120,16 +124,12 @@ namespace xt std::remove(filename.c_str()); - filename = get_filename(1); + filename = get_dump_filename(1); dump_npy(filename, ularr); auto ularrcpy = load_npy(filename); EXPECT_TRUE(all(equal(ularr, ularrcpy))); - compare_name = "files/xnpy_files/unsignedlong.npy"; - if (barr.layout() == layout_type::column_major) - { - compare_name = "files/xnpy_files/unsignedlong_fortran.npy"; - } + compare_name = get_load_filename("files/xnpy_files/unsignedlong", barr.layout()); EXPECT_TRUE(compare_binary_files(filename, compare_name)); @@ -143,7 +143,7 @@ namespace xt TEST(xnpy, xfunction_cast) { // compilation test, cf: https://github.com/xtensor-stack/xtensor/issues/1070 - auto dc = cast(load_npy("files/xnpy_files/double.npy")); + auto dc = cast(load_npy(get_load_filename("files/xnpy_files/double"))); EXPECT_EQ(dc(0, 0), 0); xarray adc = dc; EXPECT_EQ(adc(0, 0), 0); From 5d933a3ff905831e163442805be9128be1a59bff Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 8 Oct 2020 07:06:00 +0200 Subject: [PATCH 116/606] Fixed constructors of xchunked_array --- include/xtensor/xchunked_array.hpp | 48 +++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index d741bbfee..5d0ed36cf 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -22,6 +22,24 @@ namespace xt { }; + template + struct enable_chunk_store_manager + : std::enable_if::value, int> + { + }; + + template + using enable_chunk_store_manager_t = typename enable_chunk_store_manager::type; + + template + struct disable_chunk_store_manager + : std::enable_if::value, int> + { + }; + + template + using disable_chunk_store_manager_t = typename disable_chunk_store_manager::type; + /****************************** * xchunked_array declaration * ******************************/ @@ -81,12 +99,19 @@ namespace xt static constexpr layout_type static_layout = layout_type::dynamic; static constexpr bool contiguous_layout = false; - template ::value>> - xchunked_array(S&& shape, S&& chunk_shape, const std::string& directory, std::size_t pool_size = 1); - - template ::value>> + template = 0> xchunked_array(S&& shape, S&& chunk_shape); + template = 0> + xchunked_array(S&& shape, + S&& chunk_shape, + const std::string& directory, + std::size_t pool_size = 1); + ~xchunked_array() = default; xchunked_array(const xchunked_array&) = default; @@ -228,19 +253,22 @@ namespace xt *********************************/ template - template - inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape, const std::string& directory, std::size_t pool_size) + template > + inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape) { resize(std::forward(shape), std::forward(chunk_shape)); - m_chunks.set_directory(directory); - m_chunks.set_pool_size(pool_size); } template - template - inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape) + template > + inline xchunked_array::xchunked_array(S&& shape, + S&& chunk_shape, + const std::string& directory, + std::size_t pool_size) { resize(std::forward(shape), std::forward(chunk_shape)); + m_chunks.set_directory(directory); + m_chunks.set_pool_size(pool_size); } template From 79c0ee4433996362df04b6bc9466351191a758ff Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 8 Oct 2020 20:49:14 +0200 Subject: [PATCH 117/606] First implementation of zchunked_wrapper --- include/xtensor/xchunk_store_manager.hpp | 2 +- include/xtensor/xchunked_array.hpp | 2 +- include/xtensor/zarray.hpp | 7 ++ include/xtensor/zarray_impl.hpp | 121 ++++++++++++++++++++++- test/test_zarray.cpp | 12 +++ 5 files changed, 141 insertions(+), 3 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 5a1aee37a..4e5378267 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -143,7 +143,7 @@ namespace xt * xindex_path implementation * ******************************/ - void xindex_path::set_directory(const std::string& directory) + inline void xindex_path::set_directory(const std::string& directory) { m_directory = directory; if (m_directory.back() != '/') diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 5d0ed36cf..5b8fa5f27 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -400,7 +400,7 @@ namespace xt template template - inline bool xchunked_array::has_linear_assign(const S& strides) const noexcept + inline bool xchunked_array::has_linear_assign(const S&) const noexcept { return false; } diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 653d6b23f..2d698ae93 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -70,6 +70,8 @@ namespace xt template const xarray& get_array() const; + const zchunked_array& as_chunked_array() const; + private: template @@ -170,6 +172,11 @@ namespace xt { return dynamic_cast*>(p_impl.get())->get_array(); } + + inline const zchunked_array& zarray::as_chunked_array() const + { + return dynamic_cast(*(p_impl.get())); + } } #endif diff --git a/include/xtensor/zarray_impl.hpp b/include/xtensor/zarray_impl.hpp index d845f8e0b..d12ea2b89 100644 --- a/include/xtensor/zarray_impl.hpp +++ b/include/xtensor/zarray_impl.hpp @@ -11,6 +11,7 @@ #define XTENSOR_ZARRAY_IMPL_HPP #include "xarray.hpp" +#include "xchunked_array.hpp" namespace xt { @@ -154,6 +155,56 @@ namespace xt CTE m_array; }; + /******************** + * zchunked_wrapper * + ********************/ + + class zchunked_array + { + public: + + using shape_type = std::vector; + + virtual ~zchunked_array() = default; + virtual const shape_type& chunk_shape() const = 0; + }; + + template + class zchunked_wrapper : public ztyped_array::value_type>, + public zchunked_array + { + public: + + using self_type = zchunked_wrapper; + using value_type = typename std::decay_t::value_type; + using base_type = ztyped_array; + using shape_type = typename zchunked_array::shape_type; + + template + zchunked_wrapper(E&& e); + + virtual ~zchunked_wrapper() = default; + + xarray& get_array() override; + const xarray& get_array() const override; + + self_type* clone() const override; + + const shape_type& chunk_shape() const override; + + private: + + zchunked_wrapper(const zchunked_wrapper&) = default; + + void compute_cache() const; + + CTE m_chunked_array; + shape_type m_chunk_shape; + mutable xarray m_cache; + mutable bool m_cache_initialized; + + }; + /*********************** * zexpression_wrapper * ***********************/ @@ -228,6 +279,60 @@ namespace xt return new self_type(*this); } + /******************** + * zchunked_wrapper * + ********************/ + + template + template + inline zchunked_wrapper::zchunked_wrapper(E&& e) + : base_type() + , m_chunked_array(std::forward(e)) + , m_chunk_shape(m_chunked_array.chunk_shape().size()) + , m_cache() + , m_cache_initialized(false) + { + std::copy(m_chunked_array.chunk_shape().begin(), + m_chunked_array.chunk_shape().end(), + m_chunk_shape.begin()); + } + + template + inline auto zchunked_wrapper::get_array() -> xarray& + { + compute_cache(); + return m_cache; + } + + template + inline auto zchunked_wrapper::get_array() const -> const xarray& + { + compute_cache(); + return m_cache; + } + + template + inline auto zchunked_wrapper::clone() const -> self_type* + { + return new self_type(*this); + } + + template + inline auto zchunked_wrapper::chunk_shape() const -> const shape_type& + { + return m_chunk_shape; + } + + template + inline void zchunked_wrapper::compute_cache() const + { + if (!m_cache_initialized) + { + m_cache = m_chunked_array; + m_cache_initialized = true; + } + } + /****************** * zarray builder * ******************/ @@ -244,13 +349,27 @@ namespace xt { }; + template + struct is_chunked_array : std::false_type + { + }; + + template + struct is_chunked_array> : std::true_type + { + }; + template struct zwrapper_builder { using closure_type = xtl::closure_type_t; using wrapper_type = std::conditional_t>::value, zarray_wrapper, - zexpression_wrapper>; + std::conditional_t>::value, + zchunked_wrapper, + zexpression_wrapper + > + >; template static wrapper_type* run(OE&& e) diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index 133048725..9fb11ca1d 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -123,6 +123,18 @@ namespace xt const auto& res = zres.get_array(); EXPECT_TRUE(all(isclose(res, expected))); } + + TEST(zarray, chunked_array) + { + using shape_type = std::vector; + shape_type shape = {10, 10, 10}; + shape_type chunk_shape = {2, 3, 4}; + xchunked_array>> a(shape, chunk_shape); + + zarray za(a); + shape_type res = za.as_chunked_array().chunk_shape(); + EXPECT_EQ(res, chunk_shape); + } } #endif From 9517bcbb93758aefafba56c422e8a3f872004af2 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 9 Oct 2020 11:53:57 +0200 Subject: [PATCH 118/606] Don't mark dirty a resized or reshaped xfile_array --- include/xtensor/xchunked_array.hpp | 2 +- include/xtensor/xfile_array.hpp | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 5b8fa5f27..663d6b335 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -266,9 +266,9 @@ namespace xt const std::string& directory, std::size_t pool_size) { - resize(std::forward(shape), std::forward(chunk_shape)); m_chunks.set_directory(directory); m_chunks.set_pool_size(pool_size); + resize(std::forward(shape), std::forward(chunk_shape)); } template diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp index 666384061..7bc410359 100644 --- a/include/xtensor/xfile_array.hpp +++ b/include/xtensor/xfile_array.hpp @@ -403,7 +403,6 @@ namespace xt inline void xfile_array_container::resize(S&& shape, bool force) { m_storage.resize(std::forward(shape), force); - m_dirty = true; } template @@ -411,7 +410,6 @@ namespace xt inline void xfile_array_container::resize(S&& shape, layout_type l) { m_storage.resize(std::forward(shape), l); - m_dirty = true; } template @@ -419,7 +417,6 @@ namespace xt inline void xfile_array_container::resize(S&& shape, const strides_type& strides) { m_storage.resize(std::forward(shape), strides); - m_dirty = true; } template @@ -427,7 +424,6 @@ namespace xt inline auto xfile_array_container::reshape(S&& shape, layout_type layout) & -> self_type& { m_storage.reshape(std::forward(shape), layout); - m_dirty = true; return *this; } @@ -436,7 +432,6 @@ namespace xt inline auto xfile_array_container::reshape(std::initializer_list shape, layout_type layout) & -> self_type& { m_storage.reshape(shape, layout); - m_dirty = true; return *this; } From 39fcc6bf5afd646080a186b0da0eec9fbee4cd3a Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 9 Oct 2020 12:15:21 +0200 Subject: [PATCH 119/606] Replaced catch-all constructor of zarray with more restrictive ones --- include/xtensor/zarray.hpp | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp index 2d698ae93..a27ef6c2e 100644 --- a/include/xtensor/zarray.hpp +++ b/include/xtensor/zarray.hpp @@ -44,9 +44,6 @@ namespace xt zarray() = default; ~zarray() = default; - template , zarray>::value>> - zarray(E&& e); - zarray(implementation_ptr&& impl); zarray& operator=(implementation_ptr&& impl); @@ -56,6 +53,15 @@ namespace xt zarray(zarray&& rhs); zarray& operator=(zarray&& rhs); + template + zarray(const xexpression& e); + + template + zarray(xexpression& e); + + template + zarray(xexpression&& e); + template zarray& operator=(const xexpression&); @@ -99,12 +105,6 @@ namespace xt p_impl = nullptr; semantic_base::assign(e); } - - template - inline zarray::zarray(E&& e) - { - init_implementation(std::forward(e), extension::get_expression_tag_t>()); - } inline zarray::zarray(implementation_ptr&& impl) : p_impl(std::move(impl)) @@ -134,6 +134,24 @@ namespace xt { } + template + inline zarray::zarray(const xexpression& e) + { + init_implementation(e.derived_cast(), extension::get_expression_tag_t>()); + } + + template + inline zarray::zarray(xexpression& e) + { + init_implementation(e.derived_cast(), extension::get_expression_tag_t>()); + } + + template + inline zarray::zarray(xexpression&& e) + { + init_implementation(std::move(e).derived_cast(), extension::get_expression_tag_t>()); + } + inline zarray& zarray::operator=(zarray&& rhs) { swap(rhs); From 2aa535b7e545cb5b1e80a309e47d39de7ee70f3f Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 9 Oct 2020 15:08:20 +0200 Subject: [PATCH 120/606] Fixed SFINAE based on xchunked_store_manager --- include/xtensor/xchunked_array.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 663d6b335..f8b3779e2 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -16,8 +16,8 @@ namespace xt { }; - template - struct is_chunk_store_manager> + template + struct is_chunk_store_manager> : std::true_type { }; From 3375147a9fc01d20aa6572c0e6b0dcd791cf7c7a Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 9 Oct 2020 18:07:34 +0200 Subject: [PATCH 121/606] Add set_chunk_shape to the first chunk of the pool --- include/xtensor/xchunk_store_manager.hpp | 10 ++++++++++ include/xtensor/xchunked_array.hpp | 1 + 2 files changed, 11 insertions(+) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index 4e5378267..f662c986f 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -108,6 +108,9 @@ namespace xt template void resize(S&& shape); + template + void set_chunk_shape(S&& chunk_shape); + std::size_t size(); void set_pool_size(std::size_t n); @@ -260,6 +263,13 @@ namespace xt m_shape = shape; } + template + template + inline void xchunk_store_manager::set_chunk_shape(S&& chunk_shape) + { + m_chunk_pool[0].resize(chunk_shape); + } + template inline std::size_t xchunk_store_manager::size() { diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index f8b3779e2..569976b2f 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -266,6 +266,7 @@ namespace xt const std::string& directory, std::size_t pool_size) { + m_chunks.set_chunk_shape(chunk_shape); m_chunks.set_directory(directory); m_chunks.set_pool_size(pool_size); resize(std::forward(shape), std::forward(chunk_shape)); From 2ea71a5c30c1a695ae8b8cdbc1dc67d79211b53c Mon Sep 17 00:00:00 2001 From: David Brochart Date: Sat, 10 Oct 2020 10:47:17 +0200 Subject: [PATCH 122/606] Remove xchunk_store_manager setters --- include/xtensor/xchunk_store_manager.hpp | 60 +++++++----------------- include/xtensor/xchunked_array.hpp | 21 +++++---- 2 files changed, 29 insertions(+), 52 deletions(-) diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp index f662c986f..409f1102d 100644 --- a/include/xtensor/xchunk_store_manager.hpp +++ b/include/xtensor/xchunk_store_manager.hpp @@ -72,7 +72,8 @@ namespace xt using const_stepper = typename iterable_base::const_stepper; using shape_type = typename iterable_base::inner_shape_type; - xchunk_store_manager(); + template + xchunk_store_manager(S&& shape, S&& chunk_shape, const std::string& directory, std::size_t pool_size); ~xchunk_store_manager() = default; xchunk_store_manager(const xchunk_store_manager&) = default; @@ -108,13 +109,8 @@ namespace xt template void resize(S&& shape); - template - void set_chunk_shape(S&& chunk_shape); - std::size_t size(); - void set_pool_size(std::size_t n); - void set_directory(const std::string& directory); IP& get_index_path(); void flush(); @@ -175,15 +171,23 @@ namespace xt ***************************************/ template - inline xchunk_store_manager::xchunk_store_manager() - : m_shape() - // default pool size is 1 - // so that first chunk is always resized to the chunk shape - , m_chunk_pool(1u) - , m_index_pool(1u) + template + inline xchunk_store_manager::xchunk_store_manager(S&& shape, + S&& chunk_shape, + const std::string& directory, + std::size_t pool_size) + : m_shape(shape) + , m_chunk_pool(pool_size) + , m_index_pool(pool_size) , m_unload_index(0u) { - m_chunk_pool[0].ignore_empty_path(true); + // resize the pool chunks + for (auto& chunk: m_chunk_pool) + { + chunk.resize(chunk_shape); + chunk.ignore_empty_path(true); + } + m_index_path.set_directory(directory); } template @@ -263,36 +267,12 @@ namespace xt m_shape = shape; } - template - template - inline void xchunk_store_manager::set_chunk_shape(S&& chunk_shape) - { - m_chunk_pool[0].resize(chunk_shape); - } - template inline std::size_t xchunk_store_manager::size() { return compute_size(m_shape); } - template - inline void xchunk_store_manager::set_pool_size(std::size_t n) - { - // first chunk always has the correct shape - // get the shape before resizing the pool - auto chunk_shape = m_chunk_pool[0].storage().shape(); - m_chunk_pool.resize(n); - m_index_pool.resize(n); - m_unload_index = 0; - // resize the pool chunks - for (auto& chunk: m_chunk_pool) - { - chunk.resize(chunk_shape); - chunk.ignore_empty_path(true); - } - } - template inline void xchunk_store_manager::flush() { @@ -318,12 +298,6 @@ namespace xt return m_index_path; } - template - void xchunk_store_manager::set_directory(const std::string& directory) - { - m_index_path.set_directory(directory); - } - template template inline auto xchunk_store_manager::map_file_array(I first, I last) -> reference diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 569976b2f..6397794d5 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -182,7 +182,7 @@ namespace xt using dynamic_indexes_type = std::pair, std::vector>; template - void resize(S1&& shape, S2&& chunk_shape); + void resize(S1&& shape, S2&& chunk_shape, bool resize_chunks = true); template indexes_type get_indexes(Idxs... idxs) const; @@ -265,11 +265,11 @@ namespace xt S&& chunk_shape, const std::string& directory, std::size_t pool_size) + : m_chunks(shape, chunk_shape, directory, pool_size) { - m_chunks.set_chunk_shape(chunk_shape); - m_chunks.set_directory(directory); - m_chunks.set_pool_size(pool_size); - resize(std::forward(shape), std::forward(chunk_shape)); + // don't resize the "logical" chunks + // instead, the "physical" chunks in the pool are resized + resize(std::forward(shape), std::forward(chunk_shape), false); } template @@ -458,7 +458,7 @@ namespace xt template template - inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) + inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape, bool resize_chunks) { // compute chunk number in each dimension (shape_of_chunks) std::vector shape_of_chunks(shape.size()); @@ -478,10 +478,13 @@ namespace xt // resize the chunk container m_chunks.resize(shape_of_chunks); - // resize each chunk - for (auto& c: m_chunks) + if (resize_chunks) { - c.resize(chunk_shape); + // resize each chunk + for (auto& c: m_chunks) + { + c.resize(chunk_shape); + } } m_shape = xtl::forward_sequence(shape); From 85e88b5d591da6caab3caf989fc2c472b7f93b9a Mon Sep 17 00:00:00 2001 From: Gregory Lemercier Date: Mon, 12 Oct 2020 16:00:01 +0200 Subject: [PATCH 123/606] Fix generated cmake config to include missing required lib --- xtensorConfig.cmake.in | 1 + 1 file changed, 1 insertion(+) diff --git a/xtensorConfig.cmake.in b/xtensorConfig.cmake.in index 24989fa5d..833a30dfa 100644 --- a/xtensorConfig.cmake.in +++ b/xtensorConfig.cmake.in @@ -43,6 +43,7 @@ if (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} VERSION_GREATER_EQUAL 3.11) target_compile_options(xtensor::optimize INTERFACE /EHsc /MP /bigobj) # gcc, clang, ... else() + include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG(-march=native arch_native_supported) if(arch_native_supported) target_compile_options(xtensor::optimize INTERFACE -march=native) From 1f55e8574ff4e353587dc9d89cf2d3b692f69bc5 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 13 Oct 2020 11:24:20 +0200 Subject: [PATCH 124/606] Deleted files that have been moved to xtensor-io --- CMakeLists.txt | 1 + include/xtensor/xchunk_store_manager.hpp | 361 -------------- include/xtensor/xchunked_array.hpp | 4 +- include/xtensor/xdisk_io_handler.hpp | 74 --- include/xtensor/xfile_array.hpp | 607 ----------------------- test/test_xchunked_array.cpp | 96 +--- 6 files changed, 5 insertions(+), 1138 deletions(-) delete mode 100644 include/xtensor/xchunk_store_manager.hpp delete mode 100644 include/xtensor/xdisk_io_handler.hpp delete mode 100644 include/xtensor/xfile_array.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ba8dc69f1..b2e5f8465 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,7 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xbroadcast.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xbuffer_adaptor.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xbuilder.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/xchunked_array.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xcomplex.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xcontainer.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xcsv.hpp diff --git a/include/xtensor/xchunk_store_manager.hpp b/include/xtensor/xchunk_store_manager.hpp deleted file mode 100644 index 409f1102d..000000000 --- a/include/xtensor/xchunk_store_manager.hpp +++ /dev/null @@ -1,361 +0,0 @@ -#ifndef XTENSOR_CHUNK_STORE_MANAGER_HPP -#define XTENSOR_CHUNK_STORE_MANAGER_HPP - -#include -#include - -#include "xarray.hpp" -#include "xcsv.hpp" -#include "xio.hpp" - -namespace xt -{ - - /*************************** - * xindex_path declaration * - ***************************/ - - class xindex_path - { - public: - void set_directory(const std::string& directory); - template - void index_to_path(I, I, std::string&); - - private: - std::string m_directory; - }; - - /************************************ - * xchunk_store_manager declaration * - ************************************/ - - template - class xchunk_store_manager; - - template - struct xcontainer_inner_types> - { - using storage_type = EC; - using reference = EC&; - using const_reference = const EC&; - using size_type = std::size_t; - using temporary_type = xchunk_store_manager; - }; - - template - struct xiterable_inner_types> - { - using inner_shape_type = std::vector; - using stepper = xindexed_stepper, false>; - using const_stepper = xindexed_stepper, true>; - }; - - template - class xchunk_store_manager: public xaccessible>, - public xiterable> - { - public: - - using self_type = xchunk_store_manager; - using inner_types = xcontainer_inner_types; - using storage_type = typename inner_types::storage_type; - using value_type = storage_type; - using reference = EC&; - using const_reference = const EC&; - using pointer = value_type*; - using const_pointer = const value_type*; - using size_type = typename inner_types::size_type; - using difference_type = std::ptrdiff_t; - using iterable_base = xconst_iterable; - using stepper = typename iterable_base::stepper; - using const_stepper = typename iterable_base::const_stepper; - using shape_type = typename iterable_base::inner_shape_type; - - template - xchunk_store_manager(S&& shape, S&& chunk_shape, const std::string& directory, std::size_t pool_size); - ~xchunk_store_manager() = default; - - xchunk_store_manager(const xchunk_store_manager&) = default; - xchunk_store_manager& operator=(const xchunk_store_manager&) = default; - - xchunk_store_manager(xchunk_store_manager&&) = default; - xchunk_store_manager& operator=(xchunk_store_manager&&) = default; - - const shape_type& shape() const noexcept; - - template - reference operator()(Idxs... idxs); - - template - const_reference operator()(Idxs... idxs) const; - - template - reference element(It first, It last); - - template - const_reference element(It first, It last) const; - - template - stepper stepper_begin(const O& shape) noexcept; - template - stepper stepper_end(const O& shape, layout_type) noexcept; - - template - const_stepper stepper_begin(const O& shape) const noexcept; - template - const_stepper stepper_end(const O& shape, layout_type) const noexcept; - - template - void resize(S&& shape); - - std::size_t size(); - - IP& get_index_path(); - void flush(); - - template - void configure_format(C& config); - - template - reference map_file_array(I first, I last); - - template - const_reference map_file_array(I first, I last) const; - - private: - - template - std::array get_indexes(Idxs... idxs) const; - - using chunk_pool_type = std::vector; - using index_pool_type = std::vector; - - shape_type m_shape; - chunk_pool_type m_chunk_pool; - index_pool_type m_index_pool; - std::size_t m_unload_index; - IP m_index_path; - }; - - /****************************** - * xindex_path implementation * - ******************************/ - - inline void xindex_path::set_directory(const std::string& directory) - { - m_directory = directory; - if (m_directory.back() != '/') - { - m_directory.push_back('/'); - } - } - - template - void xindex_path::index_to_path(I first, I last, std::string& path) - { - std::string fname; - for (auto it = first; it != last; ++it) - { - if (!fname.empty()) - { - fname.push_back('.'); - } - fname.append(std::to_string(*it)); - } - path = m_directory + fname; - } - - /*************************************** - * xchunk_store_manager implementation * - ***************************************/ - - template - template - inline xchunk_store_manager::xchunk_store_manager(S&& shape, - S&& chunk_shape, - const std::string& directory, - std::size_t pool_size) - : m_shape(shape) - , m_chunk_pool(pool_size) - , m_index_pool(pool_size) - , m_unload_index(0u) - { - // resize the pool chunks - for (auto& chunk: m_chunk_pool) - { - chunk.resize(chunk_shape); - chunk.ignore_empty_path(true); - } - m_index_path.set_directory(directory); - } - - template - inline auto xchunk_store_manager::shape() const noexcept -> const shape_type& - { - return m_shape; - } - - template - template - inline auto xchunk_store_manager::operator()(Idxs... idxs) -> reference - { - auto index = get_indexes(idxs...); - return map_file_array(index.cbegin(), index.cend()); - } - - template - template - inline auto xchunk_store_manager::operator()(Idxs... idxs) const -> const_reference - { - auto index = get_indexes(idxs...); - return map_file_array(index.cbegin(), index.cend()); - } - - template - template - inline auto xchunk_store_manager::element(It first, It last) -> reference - { - return map_file_array(first, last); - } - - template - template - inline auto xchunk_store_manager::element(It first, It last) const -> const_reference - { - return map_file_array(first, last); - } - - template - template - inline auto xchunk_store_manager::stepper_begin(const O& shape) noexcept -> stepper - { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset); - } - - template - template - inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) noexcept -> stepper - { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset, true); - } - - template - template - inline auto xchunk_store_manager::stepper_begin(const O& shape) const noexcept -> const_stepper - { - size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset); - } - - template - template - inline auto xchunk_store_manager::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper - { - size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset, true); - } - - template - template - inline void xchunk_store_manager::resize(S&& shape) - { - // don't resize according to total number of chunks - // instead the pool manages a number of in-memory chunks - m_shape = shape; - } - - template - inline std::size_t xchunk_store_manager::size() - { - return compute_size(m_shape); - } - - template - inline void xchunk_store_manager::flush() - { - for (auto& chunk: m_chunk_pool) - { - chunk.flush(); - } - } - - template - template - void xchunk_store_manager::configure_format(C& config) - { - for (auto& chunk: m_chunk_pool) - { - chunk.configure_format(config); - } - } - - template - IP& xchunk_store_manager::get_index_path() - { - return m_index_path; - } - - template - template - inline auto xchunk_store_manager::map_file_array(I first, I last) -> reference - { - std::string path; - m_index_path.index_to_path(first, last, path); - if (first == last) - { - return m_chunk_pool[0]; - } - else - { - // check if the chunk is already loaded in memory - const auto it1 = std::find_if(m_index_pool.cbegin(), m_index_pool.cend(), [first, last](const auto& v) - { return std::equal(v.cbegin(), v.cend(), first, last); }); - std::size_t i; - if (it1 != m_index_pool.cend()) - { - i = static_cast(std::distance(m_index_pool.cbegin(), it1)); - return m_chunk_pool[i]; - } - // if not, find a free chunk in the pool - std::vector empty_index; - const auto it2 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), empty_index); - if (it2 != m_index_pool.cend()) - { - i = static_cast(std::distance(m_index_pool.cbegin(), it2)); - m_chunk_pool[i].set_path(path); - m_index_pool[i].resize(static_cast(std::distance(first, last))); - std::copy(first, last, m_index_pool[i].begin()); - return m_chunk_pool[i]; - } - // no free chunk, take one (which will thus be unloaded) - // fairness is guaranteed through the use of a walking index - m_chunk_pool[m_unload_index].set_path(path); - m_index_pool[m_unload_index].resize(static_cast(std::distance(first, last))); - std::copy(first, last, m_index_pool[m_unload_index].begin()); - auto& chunk = m_chunk_pool[m_unload_index]; - m_unload_index = (m_unload_index + 1) % m_index_pool.size(); - return chunk; - } - } - - template - template - inline auto xchunk_store_manager::map_file_array(I first, I last) const -> const_reference - { - return const_cast*>(this)->map_file_array(first, last); - } - - template - template - inline std::array - xchunk_store_manager::get_indexes(Idxs... idxs) const - { - std::array indexes = {{idxs...}}; - return indexes; - } -} - -#endif diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 6397794d5..c712121b0 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -6,11 +6,13 @@ #include "xnoalias.hpp" #include "xstrided_view.hpp" -#include "xchunk_store_manager.hpp" namespace xt { + template + class xchunk_store_manager; + template struct is_chunk_store_manager : std::false_type { diff --git a/include/xtensor/xdisk_io_handler.hpp b/include/xtensor/xdisk_io_handler.hpp deleted file mode 100644 index 47095f777..000000000 --- a/include/xtensor/xdisk_io_handler.hpp +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef XTENSOR_DISK_IO_HANDLER_HPP -#define XTENSOR_DISK_IO_HANDLER_HPP - -#include "xarray.hpp" -#include "xexpression.hpp" - -namespace xt -{ - template - class xdisk_io_handler - { - public: - - template - void write(const xexpression& expression, const std::string& path) const; - - template - void read(ET& array, const std::string& path, bool throw_on_fail = false) const; - - void configure_format(const C& format_config); - - private: - - C m_format_config; - }; - - template - template - inline void xdisk_io_handler::write(const xexpression& expression, const std::string& path) const - { - std::ofstream out_file(path, std::ofstream::binary); - if (out_file.is_open()) - { - dump_file(out_file, expression, m_format_config); - } - else - { - XTENSOR_THROW(std::runtime_error, "write: failed to open file " + path); - } - } - - template - template - inline void xdisk_io_handler::read(ET& array, const std::string& path, bool throw_on_fail) const - { - std::ifstream in_file(path, std::ifstream::binary); - if (in_file.is_open()) - { - load_file(in_file, array, m_format_config); - } - else - { - if (throw_on_fail) - { - XTENSOR_THROW(std::runtime_error, "read: failed to open file " + path); - } - else - { - auto shape = array.shape(); - array = zeros(shape); - } - } - } - - template - inline void xdisk_io_handler::configure_format(const C& format_config) - { - m_format_config = format_config; - } - - -} - -#endif diff --git a/include/xtensor/xfile_array.hpp b/include/xtensor/xfile_array.hpp deleted file mode 100644 index 7bc410359..000000000 --- a/include/xtensor/xfile_array.hpp +++ /dev/null @@ -1,607 +0,0 @@ -#ifndef XTENSOR_FILE_ARRAY_HPP -#define XTENSOR_FILE_ARRAY_HPP - -#include -#include -#include - -#include "xarray.hpp" -#include "xnoalias.hpp" - -namespace xt -{ - - template - class xfile_value_reference - { - public: - - using self_type = xfile_value_reference; - using const_reference = const T&; - - xfile_value_reference(T& value, bool& dirty); - ~xfile_value_reference() = default; - - xfile_value_reference(const xfile_value_reference&) = default; - xfile_value_reference(xfile_value_reference&&) = default; - - self_type& operator=(const self_type&); - self_type& operator=(self_type&&); - - template - self_type& operator=(const V&); - - template - self_type& operator+=(const V&); - - template - self_type& operator-=(const V&); - - template - self_type& operator*=(const V&); - - template - self_type& operator/=(const V&); - - operator const_reference() const; - - private: - - T& m_value; - bool& m_dirty; - }; -} - -namespace std -{ - template - struct is_signed> - : is_signed - { - }; -} - -namespace xt -{ - template - class xfile_array_container; - - template - struct xcontainer_inner_types> - { - using storage_type = E; - using value_type = typename storage_type::value_type; - using reference = xfile_value_reference; - using const_reference = typename storage_type::const_reference; - using size_type = typename storage_type::size_type; - using temporary_type = xfile_array_container; - }; - - template - struct xiterable_inner_types> - { - using inner_shape_type = typename E::shape_type; - using const_stepper = xindexed_stepper, true>; - using stepper = xindexed_stepper, false>; - }; - - template - class xfile_array_container : public xaccessible>, - public xiterable>, - public xcontainer_semantic> - { - public: - - using self_type = xfile_array_container; - using semantic_base = xcontainer_semantic; - using iterable_base = xconst_iterable; - using inner_types = xcontainer_inner_types; - using storage_type = typename inner_types::storage_type; - using value_type = typename storage_type::value_type; - using reference = typename inner_types::reference; - using const_reference = typename inner_types::const_reference; - using pointer = typename storage_type::pointer; - using const_pointer = typename storage_type::const_pointer; - using size_type = typename inner_types::size_type; - using difference_type = typename storage_type::difference_type; - using shape_type = typename storage_type::shape_type; - using strides_type = typename storage_type::strides_type; - using stepper = typename iterable_base::stepper; - using const_stepper = typename iterable_base::const_stepper; - using temporary_type = typename inner_types::temporary_type; - using bool_load_type = xt::bool_load_type; - static constexpr layout_type static_layout = layout_type::dynamic; - static constexpr bool contiguous_layout = true; - - xfile_array_container() = default; - ~xfile_array_container(); - - xfile_array_container(const self_type&) = default; - self_type& operator=(const self_type&) = default; - - xfile_array_container(self_type&&) = default; - self_type& operator=(self_type&&) = default; - - template - xfile_array_container(const xexpression& e); - - template - xfile_array_container(const xexpression& e, const std::string& path); - - template - self_type& operator=(const xexpression& e); - - size_type size() const noexcept; - const shape_type& shape() const noexcept; - layout_type layout() const noexcept; - bool is_contiguous() const noexcept; - - template - void resize(S&& shape, bool force = false); - template - void resize(S&& shape, layout_type l); - template - void resize(S&& shape, const strides_type& strides); - - template - self_type& reshape(S&& shape, layout_type layout = static_layout) &; - - template - self_type& reshape(std::initializer_list shape, layout_type layout = static_layout) &; - - template - reference operator()(Idxs... idxs); - - template - const_reference operator()(Idxs... idxs) const; - - template - reference element(It first, It last); - - template - const_reference element(It first, It last) const; - - storage_type& storage() noexcept; - const storage_type& storage() const noexcept; - - template - bool broadcast_shape(S& s, bool reuse_cache = false) const; - - template - bool has_linear_assign(const S& strides) const noexcept; - - template - stepper stepper_begin(const O& shape) noexcept; - template - stepper stepper_end(const O& shape, layout_type) noexcept; - - template - const_stepper stepper_begin(const O& shape) const noexcept; - template - const_stepper stepper_end(const O& shape, layout_type) const noexcept; - - reference data_element(size_type i); - const_reference data_element(size_type i) const; - - template - using simd_return_type = xt_simd::simd_return_type; - - template - void store_simd(size_type i, const simd& e); - template ::size> - container_simd_return_type_t - load_simd(size_type i) const; - - const std::string& path() const noexcept; - void ignore_empty_path(bool ignore); - void set_path(const std::string& path); - - template - void configure_format(C& config); - - void flush(); - - private: - - bool enable_io(const std::string& path) const; - - E m_storage; - bool m_dirty; - IOH m_io_handler; - std::string m_path; - bool m_ignore_empty_path; - }; - - template ::size_type>> - using xfile_array = xfile_array_container, IOH>; - - /**************************************** - * xfile_value_reference implementation * - ****************************************/ - - template - inline xfile_value_reference::xfile_value_reference(T& value, bool& dirty) - : m_value(value), m_dirty(dirty) - { - } - - template - template - inline auto xfile_value_reference::operator=(const V& v) -> self_type& - { - if (v != m_value) - { - m_value = v; - m_dirty = true; - } - return *this; - } - - template - template - inline auto xfile_value_reference::operator+=(const V& v) -> self_type& - { - if (v != T(0)) - { - m_value += v; - m_dirty = true; - } - return *this; - } - - template - template - inline auto xfile_value_reference::operator-=(const V& v) -> self_type& - { - if (v != T(0)) - { - m_value -= v; - m_dirty = true; - } - return *this; - } - - template - template - inline auto xfile_value_reference::operator*=(const V& v) -> self_type& - { - if (v != T(1)) - { - m_value *= v; - m_dirty = true; - } - return *this; - } - - template - template - inline auto xfile_value_reference::operator/=(const V& v) -> self_type& - { - if (v != T(1)) - { - m_value /= v; - m_dirty = true; - } - return *this; - } - - template - inline xfile_value_reference::operator const_reference() const - { - return m_value; - } - - /**************************************** - * xfile_array_container implementation * - ****************************************/ - - namespace detail - { - // Workaround for VS2015 - template - using try_path = decltype(std::declval().path()); - - template class OP, class = void> - struct file_helper_impl - { - using is_stored = std::false_type; - - static const char* path(const xexpression&) - { - return ""; - } - }; - - template class OP> - struct file_helper_impl>> - { - using is_stored = std::true_type; - - static const char* path(const xexpression& e) - { - return e.derived_cast().path(); - } - }; - - template - using file_helper = file_helper_impl; - } - - template - constexpr bool is_stored(const xexpression&) - { - using return_type = typename detail::file_helper::is_stored; - return return_type::value; - } - - template - inline xfile_array_container::~xfile_array_container() - { - flush(); - } - - template - template - inline xfile_array_container::xfile_array_container(const xexpression& e) - : m_storage(e) - , m_dirty(true) - , m_io_handler() - , m_path(detail::file_helper::path(e)) - , m_ignore_empty_path(false) - { - } - - template - template - inline xfile_array_container::xfile_array_container(const xexpression& e, const std::string& path) - : m_storage(e) - , m_dirty(true) - , m_io_handler() - , m_path(path) - , m_ignore_empty_path(false) - { - } - - template - template - inline auto xfile_array_container::operator=(const xexpression& e) -> self_type& - { - return semantic_base::operator=(e); - } - - template - inline auto xfile_array_container::size() const noexcept -> size_type - { - return m_storage.size(); - } - - template - inline auto xfile_array_container::shape() const noexcept -> const shape_type& - { - return m_storage.shape(); - } - - template - inline auto xfile_array_container::layout() const noexcept -> layout_type - { - return m_storage.layout(); - } - - template - inline bool xfile_array_container::is_contiguous() const noexcept - { - return m_storage.is_contiguous(); - } - - template - template - inline void xfile_array_container::resize(S&& shape, bool force) - { - m_storage.resize(std::forward(shape), force); - } - - template - template - inline void xfile_array_container::resize(S&& shape, layout_type l) - { - m_storage.resize(std::forward(shape), l); - } - - template - template - inline void xfile_array_container::resize(S&& shape, const strides_type& strides) - { - m_storage.resize(std::forward(shape), strides); - } - - template - template - inline auto xfile_array_container::reshape(S&& shape, layout_type layout) & -> self_type& - { - m_storage.reshape(std::forward(shape), layout); - return *this; - } - - template - template - inline auto xfile_array_container::reshape(std::initializer_list shape, layout_type layout) & -> self_type& - { - m_storage.reshape(shape, layout); - return *this; - } - - template - template - inline auto xfile_array_container::operator()(Idxs... idxs) -> reference - { - return reference(m_storage(idxs...), m_dirty); - } - - template - template - inline auto xfile_array_container::operator()(Idxs... idxs) const -> const_reference - { - return m_storage(idxs...); - } - - template - template - inline auto xfile_array_container::element(It first, It last) -> reference - { - return reference(m_storage.element(first, last), m_dirty); - } - - template - template - inline auto xfile_array_container::element(It first, It last) const -> const_reference - { - return m_storage.element(first, last); - } - - template - inline auto xfile_array_container::storage() noexcept -> storage_type& - { - return m_storage; - } - - template - inline auto xfile_array_container::storage() const noexcept -> const storage_type& - { - return m_storage; - } - - template - template - inline bool xfile_array_container::broadcast_shape(S& s, bool reuse_cache) const - { - return m_storage.broadcast_shape(s, reuse_cache); - } - - template - template - inline bool xfile_array_container::has_linear_assign(const S& strides) const noexcept - { - return m_storage.has_linear_assign(strides); - } - - template - template - inline auto xfile_array_container::stepper_begin(const O& shape) noexcept -> stepper - { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset); - } - - template - template - inline auto xfile_array_container::stepper_end(const O& shape, layout_type) noexcept -> stepper - { - size_type offset = shape.size() - this->dimension(); - return stepper(this, offset, true); - } - - template - template - inline auto xfile_array_container::stepper_begin(const O& shape) const noexcept -> const_stepper - { - size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset); - } - - template - template - inline auto xfile_array_container::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper - { - size_type offset = shape.size() - this->dimension(); - return const_stepper(this, offset, true); - } - - template - inline auto xfile_array_container::data_element(size_type i) -> reference - { - return reference(m_storage.data_element(i), m_dirty); - } - - template - inline auto xfile_array_container::data_element(size_type i) const -> const_reference - { - return m_storage.element(i); - } - - template - template - inline void xfile_array_container::store_simd(size_type i, const simd& e) - { - m_storage.store_simd(i, e); - m_dirty = true; - } - - template - template - inline auto xfile_array_container::load_simd(size_type i) const - -> container_simd_return_type_t - { - return m_storage.load_simd(i); - } - - template - inline const std::string& xfile_array_container::path() const noexcept - { - return m_path; - } - - template - template - inline void xfile_array_container::configure_format(C& config) - { - m_io_handler.configure_format(config); - } - - template - inline void xfile_array_container::ignore_empty_path(bool ignore) - { - m_ignore_empty_path = ignore; - } - - template - inline bool xfile_array_container::enable_io(const std::string& path) const - { - return !path.empty() || !m_ignore_empty_path; - } - - template - inline void xfile_array_container::set_path(const std::string& path) - { - if (path != m_path) - { - // maybe write to old file - flush(); - m_path = path; - // read new file - if (enable_io(path)) - { - m_io_handler.read(m_storage, path); - } - } - } - - template - inline void xfile_array_container::flush() - { - if (m_dirty) - { - if (enable_io(m_path)) - { - m_io_handler.write(m_storage, m_path); - } - m_dirty = false; - } - } -} - -#endif diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index 85ee4c781..d16f006b2 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -9,11 +9,9 @@ #include "gtest/gtest.h" +#include "xtensor/xarray.hpp" #include "xtensor/xbroadcast.hpp" #include "xtensor/xchunked_array.hpp" -#include "xtensor/xchunk_store_manager.hpp" -#include "xtensor/xfile_array.hpp" -#include "xtensor/xdisk_io_handler.hpp" #include "xtensor/xcsv.hpp" namespace xt @@ -48,9 +46,6 @@ namespace xt TEST(xchunked_array, assign_expression) { -#ifdef _MSC_FULL_VER - std::cout << "MSC_FULL_VER = " << _MSC_FULL_VER << std::endl; -#endif std::vector shape1 = {2, 2, 2}; std::vector chunk_shape1 = {2, 3, 4}; in_memory_chunked_array a1(shape1, chunk_shape1); @@ -111,93 +106,4 @@ namespace xt EXPECT_EQ(v, 3); } } - - TEST(xchunked_array, disk_array) - { - std::vector shape = {4, 4}; - std::vector chunk_shape = {2, 2}; - std::string chunk_dir = "files"; - std::size_t pool_size = 2; - xchunked_array>>> a1(shape, chunk_shape, chunk_dir, pool_size); - std::vector idx = {1, 2}; - double v1 = 3.4; - double v2 = 5.6; - double v3 = 7.8; - a1(2, 1) = v1; - a1[idx] = v2; - a1(0, 0) = v3; // this should unload chunk 1.0 - ASSERT_EQ(a1(2, 1), v1); - ASSERT_EQ(a1[idx], v2); - ASSERT_EQ(a1(0, 0), v3); - - std::ifstream in_file; - xt::xarray ref; - xt::xarray data; - in_file.open(chunk_dir + "/1.0"); - data = xt::load_csv(in_file); - ref = {{0, v1}, {0, 0}}; - EXPECT_EQ(data, ref); - in_file.close(); - - a1.chunks().flush(); - in_file.open(chunk_dir + "/0.1"); - data = xt::load_csv(in_file); - ref = {{0, 0}, {v2, 0}}; - EXPECT_EQ(data, ref); - in_file.close(); - - in_file.open(chunk_dir + "/0.0"); - data = xt::load_csv(in_file); - ref = {{v3, 0}, {0, 0}}; - EXPECT_EQ(data, ref); - in_file.close(); - } - - TEST(xfile_array, indexed_access) - { - std::vector shape = {2, 2, 2}; - xfile_array> a; - a.ignore_empty_path(true); - a.resize(shape); - double val = 3.; - for (auto it: a) - it = val; - for (auto it: a) - ASSERT_EQ(it, val); - } - - TEST(xfile_array, assign_expression) - { - double v1 = 3.; - auto a1 = xfile_array>(broadcast(v1, {2, 2}), "a1"); - a1.ignore_empty_path(true); - for (const auto& v: a1) - { - EXPECT_EQ(v, v1); - } - - double v2 = 2. * v1; - auto a2 = xfile_array>(a1 + a1, "a2"); - a2.ignore_empty_path(true); - for (const auto& v: a2) - { - EXPECT_EQ(v, v2); - } - - a1.flush(); - a2.flush(); - - std::ifstream in_file; - in_file.open("a1"); - auto data = load_csv(in_file); - xarray ref = {{v1, v1}, {v1, v1}}; - EXPECT_EQ(data, ref); - in_file.close(); - - in_file.open("a2"); - data = load_csv(in_file); - ref = {{v2, v2}, {v2, v2}}; - EXPECT_EQ(data, ref); - in_file.close(); - } } From 3a9bd7c147ffaacff63e7db72cedbca2091fc9f2 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 13 Oct 2020 21:46:05 +0200 Subject: [PATCH 125/606] Refactored xchunked_array constructors --- include/xtensor/xchunked_array.hpp | 107 ++++++++++------------------- test/test_xchunked_array.cpp | 7 +- test/test_zarray.cpp | 2 +- 3 files changed, 39 insertions(+), 77 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index c712121b0..3b93a5069 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -4,44 +4,13 @@ #include #include +#include "xarray.hpp" #include "xnoalias.hpp" #include "xstrided_view.hpp" namespace xt { - template - class xchunk_store_manager; - - template - struct is_chunk_store_manager : std::false_type - { - }; - - template - struct is_chunk_store_manager> - : std::true_type - { - }; - - template - struct enable_chunk_store_manager - : std::enable_if::value, int> - { - }; - - template - using enable_chunk_store_manager_t = typename enable_chunk_store_manager::type; - - template - struct disable_chunk_store_manager - : std::enable_if::value, int> - { - }; - - template - using disable_chunk_store_manager_t = typename disable_chunk_store_manager::type; - /****************************** * xchunked_array declaration * ******************************/ @@ -101,19 +70,8 @@ namespace xt static constexpr layout_type static_layout = layout_type::dynamic; static constexpr bool contiguous_layout = false; - template = 0> - xchunked_array(S&& shape, S&& chunk_shape); - - template = 0> - xchunked_array(S&& shape, - S&& chunk_shape, - const std::string& directory, - std::size_t pool_size = 1); - + template + xchunked_array(chunk_storage_type&& chunks, S&& shape, S&& chunk_shape); ~xchunked_array() = default; xchunked_array(const xchunked_array&) = default; @@ -184,7 +142,7 @@ namespace xt using dynamic_indexes_type = std::pair, std::vector>; template - void resize(S1&& shape, S2&& chunk_shape, bool resize_chunks = true); + void resize(S1&& shape, S2&& chunk_shape); template indexes_type get_indexes(Idxs... idxs) const; @@ -209,6 +167,9 @@ namespace xt template constexpr bool is_chunked(const xexpression& e); + template + xchunked_array>> chunked_array(S&& shape, S&& chunk_shape); + /******************************* * chunk_helper implementation * *******************************/ @@ -227,6 +188,16 @@ namespace xt { return e.derived_cast().shape(); } + + template + static void resize(E& chunks, const S1& container_shape, const S2& chunk_shape) + { + chunks.resize(container_shape); + for(auto& c: chunks) + { + c.resize(chunk_shape); + } + } }; template class OP> @@ -237,6 +208,12 @@ namespace xt { return e.derived_cast().chunk_shape(); } + + template + static void resize(E& chunks, const S1& container_shape, const S2& chunk_shape) + { + chunks.resize(container_shape, chunk_shape); + } }; template @@ -250,30 +227,25 @@ namespace xt return return_type::value; } + template + inline xchunked_array>> chunked_array(S&& shape, S&& chunk_shape) + { + using chunk_storage = xarray>; + return xchunked_array(chunk_storage(), std::forward(shape), std::forward(chunk_shape)); + } + /********************************* * xchunked_array implementation * *********************************/ template - template > - inline xchunked_array::xchunked_array(S&& shape, S&& chunk_shape) + template + inline xchunked_array::xchunked_array(CS&& chunks, S&& shape, S&& chunk_shape) + : m_chunks(chunks) { resize(std::forward(shape), std::forward(chunk_shape)); } - template - template > - inline xchunked_array::xchunked_array(S&& shape, - S&& chunk_shape, - const std::string& directory, - std::size_t pool_size) - : m_chunks(shape, chunk_shape, directory, pool_size) - { - // don't resize the "logical" chunks - // instead, the "physical" chunks in the pool are resized - resize(std::forward(shape), std::forward(chunk_shape), false); - } - template template inline xchunked_array::xchunked_array(const xexpression& e) @@ -460,7 +432,7 @@ namespace xt template template - inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape, bool resize_chunks) + inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) { // compute chunk number in each dimension (shape_of_chunks) std::vector shape_of_chunks(shape.size()); @@ -478,16 +450,7 @@ namespace xt } ); - // resize the chunk container - m_chunks.resize(shape_of_chunks); - if (resize_chunks) - { - // resize each chunk - for (auto& c: m_chunks) - { - c.resize(chunk_shape); - } - } + detail::chunk_helper::resize(m_chunks, shape_of_chunks, chunk_shape); m_shape = xtl::forward_sequence(shape); m_chunk_shape = xtl::forward_sequence(chunk_shape); diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index d16f006b2..c3d65dcd7 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -9,7 +9,6 @@ #include "gtest/gtest.h" -#include "xtensor/xarray.hpp" #include "xtensor/xbroadcast.hpp" #include "xtensor/xchunked_array.hpp" #include "xtensor/xcsv.hpp" @@ -22,7 +21,7 @@ namespace xt { std::vector shape = {10, 10, 10}; std::vector chunk_shape = {2, 3, 4}; - in_memory_chunked_array a(shape, chunk_shape); + auto a = chunked_array(shape, chunk_shape); std::vector idx = {3, 9, 8}; double val; @@ -48,7 +47,7 @@ namespace xt { std::vector shape1 = {2, 2, 2}; std::vector chunk_shape1 = {2, 3, 4}; - in_memory_chunked_array a1(shape1, chunk_shape1); + auto a1 = chunked_array(shape1, chunk_shape1); double val; val = 3.; @@ -59,7 +58,7 @@ namespace xt } std::vector shape2 = {32, 10, 10}; - in_memory_chunked_array a2(shape2, chunk_shape1); + auto a2 = chunked_array(shape2, chunk_shape1); a2 = broadcast(val, a2.shape()); for (const auto& v: a2) diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp index 9fb11ca1d..6597d866b 100644 --- a/test/test_zarray.cpp +++ b/test/test_zarray.cpp @@ -129,7 +129,7 @@ namespace xt using shape_type = std::vector; shape_type shape = {10, 10, 10}; shape_type chunk_shape = {2, 3, 4}; - xchunked_array>> a(shape, chunk_shape); + auto a = chunked_array(shape, chunk_shape); zarray za(a); shape_type res = za.as_chunked_array().chunk_shape(); From 5e370fa4665dd16bdd67a28f5d8af7ed2d42eea7 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 14 Oct 2020 16:50:17 +0200 Subject: [PATCH 126/606] Refactored xchunked_array semantic --- include/xtensor/xchunked_array.hpp | 220 +++++++++++++++++++++++------ include/xtensor/xsemantic.hpp | 16 +-- test/test_xchunked_array.cpp | 11 +- 3 files changed, 196 insertions(+), 51 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 3b93a5069..97f89d69d 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -11,6 +11,59 @@ namespace xt { + /********************************* + * xchunked_semantic declaration * + *********************************/ + + template + class xchunked_assigner + { + public: + + using temporary_type = T; + + template + void build_and_assign_temporary(const xexpression& e, DST& dst); + }; + + template + class xchunked_semantic : public xsemantic_base + { + public: + + using base_type = xsemantic_base; + using derived_type = D; + using temporary_type = typename base_type::temporary_type; + + template + derived_type& assign_xexpression(const xexpression& e); + + template + derived_type& computed_assign(const xexpression& e); + + template + derived_type& scalar_computed_assign(const E& e, F&& f); + + protected: + + xchunked_semantic() = default; + ~xchunked_semantic() = default; + + xchunked_semantic(const xchunked_semantic&) = default; + xchunked_semantic& operator=(const xchunked_semantic&) = default; + + xchunked_semantic(xchunked_semantic&&) = default; + xchunked_semantic& operator=(xchunked_semantic&&) = default; + + template + derived_type& operator=(const xexpression& e); + + private: + + template + xchunked_assigner get_assigner(const CS&) const; + }; + /****************************** * xchunked_array declaration * ******************************/ @@ -43,7 +96,7 @@ namespace xt template class xchunked_array: public xaccessible>, public xiterable>, - public xcontainer_semantic>, + public xchunked_semantic>, public extension { public: @@ -53,7 +106,7 @@ namespace xt using const_reference = typename chunk_type::const_reference; using reference = typename chunk_type::reference; using self_type = xchunked_array; - using semantic_base = xcontainer_semantic; + using semantic_base = xchunked_semantic; using iterable_base = xconst_iterable; using const_stepper = typename iterable_base::const_stepper; using stepper = typename iterable_base::stepper; @@ -81,10 +134,10 @@ namespace xt xchunked_array& operator=(xchunked_array&&) = default; template - xchunked_array(const xexpression& e); + xchunked_array(const xexpression&e , chunk_storage_type&& chunks); template - xchunked_array(const xexpression& e, S&& chunk_shape); + xchunked_array(const xexpression& e, chunk_storage_type&& chunks, S&& chunk_shape); template xchunked_array& operator=(const xexpression& e); @@ -167,8 +220,16 @@ namespace xt template constexpr bool is_chunked(const xexpression& e); - template - xchunked_array>> chunked_array(S&& shape, S&& chunk_shape); + template + xchunked_array>, EXT> chunked_array(S&& shape, S&& chunk_shape); + + template + xchunked_array>, EXT> + chunked_array(const xexpression& e, S&& chunk_shape); + + template + xchunked_array>, EXT> + chunked_array(const xexpression&e); /******************************* * chunk_helper implementation * @@ -227,62 +288,68 @@ namespace xt return return_type::value; } - template - inline xchunked_array>> chunked_array(S&& shape, S&& chunk_shape) + template + inline xchunked_array>, EXT> chunked_array(S&& shape, S&& chunk_shape) { using chunk_storage = xarray>; - return xchunked_array(chunk_storage(), std::forward(shape), std::forward(chunk_shape)); + return xchunked_array(chunk_storage(), std::forward(shape), std::forward(chunk_shape)); } - /********************************* - * xchunked_array implementation * - *********************************/ - - template - template - inline xchunked_array::xchunked_array(CS&& chunks, S&& shape, S&& chunk_shape) - : m_chunks(chunks) + template + inline xchunked_array>, EXT> + chunked_array(const xexpression& e, S&& chunk_shape) { - resize(std::forward(shape), std::forward(chunk_shape)); + using chunk_storage = xarray>; + return xchunked_array(e, chunk_storage(), std::forward(chunk_shape)); } - template - template - inline xchunked_array::xchunked_array(const xexpression& e) - : xchunked_array(e, detail::chunk_helper::chunk_shape(e)) + template + inline xchunked_array>, EXT> + chunked_array(const xexpression& e) { + using chunk_storage = xarray>; + return xchunked_array(e, chunk_storage()); } - template - template - inline xchunked_array::xchunked_array(const xexpression& e, S&& chunk_shape) + /************************************ + * xchunked_semantic implementation * + ************************************/ + + template + template + inline void xchunked_assigner::build_and_assign_temporary(const xexpression& e, DST& dst) { - resize(e.derived_cast().shape(), std::forward(chunk_shape)); - assign(e); + temporary_type tmp(e, CS(), dst.chunk_shape()); + dst = std::move(tmp); } - template + template template - inline void xchunked_array::assign(const xexpression& e) - { - xstrided_slice_vector sv(m_chunk_shape.size()); // element slice corresponding to chunk - std::transform(m_chunk_shape.begin(), m_chunk_shape.end(), sv.begin(), + inline auto xchunked_semantic::assign_xexpression(const xexpression& e) -> derived_type& + { + using shape_type = std::decay_tderived_cast().shape())>; + using size_type = typename shape_type::size_type; + const auto& chunk_shape = this->derived_cast().chunk_shape(); + auto& chunks = this->derived_cast().chunks(); + size_t dimension = this->derived_cast().dimension(); + xstrided_slice_vector sv(chunk_shape.size()); // element slice corresponding to chunk + std::transform(chunk_shape.begin(), chunk_shape.end(), sv.begin(), [](auto size) { return range(0, size); }); - shape_type ic(this->dimension()); // index of chunk, initialized to 0... + shape_type ic(dimension); // index of chunk, initialized to 0... size_type ci = 0; - for (auto& chunk: m_chunks) + for (auto& chunk: chunks) { noalias(chunk) = strided_view(e.derived_cast(), sv); - bool last_chunk = ci == m_chunks.size() - 1; + bool last_chunk = ci == chunks.size() - 1; if (!last_chunk) { - size_type di = this->dimension() - 1; + size_type di = dimension - 1; while (true) { - if (ic[di] + 1 == m_chunks.shape()[di]) + if (ic[di] + 1 == chunks.shape()[di]) { ic[di] = 0; - sv[di] = range(0, m_chunk_shape[di]); + sv[di] = range(0, chunk_shape[di]); if (di == 0) { break; @@ -295,21 +362,92 @@ namespace xt else { ic[di] += 1; - sv[di] = range(ic[di] * m_chunk_shape[di], (ic[di] + 1) * m_chunk_shape[di]); + sv[di] = range(ic[di] * chunk_shape[di], (ic[di] + 1) * chunk_shape[di]); break; } } } ++ci; } + return this->derived_cast(); + } + + template + template + inline auto xchunked_semantic::computed_assign(const xexpression& e) -> derived_type& + { + D& d = this->derived_cast(); + if (e.derived_cast().dimension() > d.dimension() + || e.derived_cast().shape() > d.shape()) + { + return operator=(e); + } + else + { + return assign_xexpression(e); + } + } + + template + template + inline auto xchunked_semantic::scalar_computed_assign(const E& e, F&& f) -> derived_type& + { + for (auto& c: this->derived_cast().chunks()) + { + c.scalar_computed_assign(e, f); + } + return this->derived_cast(); + } + + template + template + inline auto xchunked_semantic::operator=(const xexpression& e) -> derived_type& + { + D& d = this->derived_cast(); + get_assigner(d.chunks()).build_and_assign_temporary(e, d); + return d; + } + + template + template + inline auto xchunked_semantic::get_assigner(const CS&) const -> xchunked_assigner + { + return xchunked_assigner(); + } + + /********************************* + * xchunked_array implementation * + *********************************/ + + template + template + inline xchunked_array::xchunked_array(CS&& chunks, S&& shape, S&& chunk_shape) + : m_chunks(std::move(chunks)) + { + resize(std::forward(shape), std::forward(chunk_shape)); + } + + template + template + inline xchunked_array::xchunked_array(const xexpression& e, CS&& chunks) + : xchunked_array(e, std::move(chunks), detail::chunk_helper::chunk_shape(e)) + { + } + + template + template + inline xchunked_array::xchunked_array(const xexpression& e, CS&& chunks, S&& chunk_shape) + : m_chunks(std::move(chunks)) + { + resize(e.derived_cast().shape(), std::forward(chunk_shape)); + semantic_base::assign_xexpression(e); } template template inline auto xchunked_array::operator=(const xexpression& e) -> self_type& { - assign(e); - return *this; + return semantic_base::operator=(e); } template diff --git a/include/xtensor/xsemantic.hpp b/include/xtensor/xsemantic.hpp index 1479ef76c..2f0b233eb 100644 --- a/include/xtensor/xsemantic.hpp +++ b/include/xtensor/xsemantic.hpp @@ -385,7 +385,7 @@ namespace xt template inline auto xsemantic_base::operator+=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() + e.derived_cast()); + return this->derived_cast() = this->derived_cast() + e.derived_cast(); } /** @@ -397,7 +397,7 @@ namespace xt template inline auto xsemantic_base::operator-=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() - e.derived_cast()); + return this->derived_cast() = this->derived_cast() - e.derived_cast(); } /** @@ -409,7 +409,7 @@ namespace xt template inline auto xsemantic_base::operator*=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() * e.derived_cast()); + return this->derived_cast() = this->derived_cast() * e.derived_cast(); } /** @@ -421,7 +421,7 @@ namespace xt template inline auto xsemantic_base::operator/=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() / e.derived_cast()); + return this->derived_cast() = this->derived_cast() / e.derived_cast(); } /** @@ -433,7 +433,7 @@ namespace xt template inline auto xsemantic_base::operator%=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() % e.derived_cast()); + return this->derived_cast() = this->derived_cast() % e.derived_cast(); } /** @@ -445,7 +445,7 @@ namespace xt template inline auto xsemantic_base::operator&=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() & e.derived_cast()); + return this->derived_cast() = this->derived_cast() & e.derived_cast(); } /** @@ -457,7 +457,7 @@ namespace xt template inline auto xsemantic_base::operator|=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() | e.derived_cast()); + return this->derived_cast() = this->derived_cast() | e.derived_cast(); } /** @@ -469,7 +469,7 @@ namespace xt template inline auto xsemantic_base::operator^=(const xexpression& e) -> derived_type& { - return operator=(this->derived_cast() ^ e.derived_cast()); + return this->derived_cast() = this->derived_cast() ^ e.derived_cast(); } //@} diff --git a/test/test_xchunked_array.cpp b/test/test_xchunked_array.cpp index c3d65dcd7..9b5211e04 100644 --- a/test/test_xchunked_array.cpp +++ b/test/test_xchunked_array.cpp @@ -72,6 +72,13 @@ namespace xt EXPECT_EQ(v, 2. * val); } + a2 += 2.; + for (const auto& v: a2) + { + EXPECT_EQ(v, 2. * val + 2.); + } + + xarray a3 {{1., 2., 3.}, {4., 5., 6.}, @@ -80,7 +87,7 @@ namespace xt EXPECT_EQ(is_chunked(a3), false); std::vector chunk_shape4 = {2, 2}; - auto a4 = in_memory_chunked_array(a3, chunk_shape4); + auto a4 = chunked_array(a3, chunk_shape4); EXPECT_EQ(is_chunked(a4), true); @@ -98,7 +105,7 @@ namespace xt EXPECT_EQ(v, 2); } - auto a6 = in_memory_chunked_array(a3); + auto a6 = chunked_array(a3); EXPECT_EQ(is_chunked(a6), true); for (const auto& v: a6.chunk_shape()) { From 337f1daaee552f798f1f3636b849c72c7176bd85 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 14 Oct 2020 23:39:24 +0200 Subject: [PATCH 127/606] Added missing header to CMakeLists.txt --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b2e5f8465..64dfd0755 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -182,6 +182,7 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xview_utils.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zarray.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zarray_impl.hpp + ${XTENSOR_INCLUDE_DIR}/xtensor/zassign.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zdispatcher.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zdispatching_types.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/zfunction.hpp From ce9fc55af5074a5c6f5bbe478c142cd75655b404 Mon Sep 17 00:00:00 2001 From: 0xBYTESHIFT <0xBYTESHIFT@gmail.com> Date: Thu, 15 Oct 2020 04:38:40 +0300 Subject: [PATCH 128/606] changed std traits to new xtl::xtraits --- include/xtensor/xadapt.hpp | 20 +++++------ include/xtensor/xassign.hpp | 6 ++-- include/xtensor/xbuilder.hpp | 8 ++--- include/xtensor/xcontainer.hpp | 4 +-- include/xtensor/xgenerator.hpp | 6 ++-- include/xtensor/xhistogram.hpp | 4 +-- include/xtensor/xio.hpp | 6 ++-- include/xtensor/xmanipulation.hpp | 4 +-- include/xtensor/xmath.hpp | 40 ++++++++++----------- include/xtensor/xnorm.hpp | 4 +-- include/xtensor/xpad.hpp | 4 +-- include/xtensor/xrandom.hpp | 4 +-- include/xtensor/xslice.hpp | 60 +++++++++++++++---------------- include/xtensor/xsort.hpp | 8 ++--- include/xtensor/xstrides.hpp | 2 +- include/xtensor/xutils.hpp | 14 ++++---- include/xtensor/xview.hpp | 4 +-- include/xtensor/xview_utils.hpp | 6 ++-- 18 files changed, 102 insertions(+), 102 deletions(-) diff --git a/include/xtensor/xadapt.hpp b/include/xtensor/xadapt.hpp index 21cc7e5fb..8b18e55a4 100644 --- a/include/xtensor/xadapt.hpp +++ b/include/xtensor/xadapt.hpp @@ -74,7 +74,7 @@ namespace xt inline xarray_adaptor, L, std::decay_t> adapt(C&& container, const SC& shape, layout_type l = L) { - static_assert(!std::is_integral::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral::value, "shape cannot be a integer"); using return_type = xarray_adaptor, L, std::decay_t>; return return_type(std::forward(container), shape, l); } @@ -90,7 +90,7 @@ namespace xt std::is_pointer)> inline auto adapt(C&& pointer, const SC& shape, layout_type l = L) { - static_assert(!std::is_integral::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral::value, "shape cannot be a integer"); using buffer_type = xbuffer_adaptor>; using return_type = xarray_adaptor>; std::size_t size = compute_size(shape); @@ -110,7 +110,7 @@ namespace xt inline xarray_adaptor, layout_type::dynamic, std::decay_t> adapt(C&& container, SC&& shape, SS&& strides) { - static_assert(!std::is_integral>::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral>::value, "shape cannot be a integer"); using return_type = xarray_adaptor, layout_type::dynamic, std::decay_t>; return return_type(std::forward(container), xtl::forward_sequence(shape), @@ -133,7 +133,7 @@ namespace xt inline xarray_adaptor, O, A>, L, SC> adapt(P&& pointer, typename A::size_type size, O ownership, const SC& shape, layout_type l = L, const A& alloc = A()) { - static_assert(!std::is_integral::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; using return_type = xarray_adaptor; @@ -158,7 +158,7 @@ namespace xt inline xarray_adaptor, O, A>, layout_type::dynamic, std::decay_t> adapt(P&& pointer, typename A::size_type size, O ownership, SC&& shape, SS&& strides, const A& alloc = A()) { - static_assert(!std::is_integral>::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral>::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; using return_type = xarray_adaptor>; @@ -231,7 +231,7 @@ namespace xt inline xtensor_adaptor::value, L> adapt(C&& container, const SC& shape, layout_type l = L) { - static_assert(!std::is_integral::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral::value, "shape cannot be a integer"); constexpr std::size_t N = detail::array_size::value; using return_type = xtensor_adaptor, N, L>; return return_type(std::forward(container), shape, l); @@ -248,7 +248,7 @@ namespace xt std::is_pointer)> inline auto adapt(C&& pointer, const SC& shape, layout_type l = L) { - static_assert(!std::is_integral::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral::value, "shape cannot be a integer"); using buffer_type = xbuffer_adaptor>; constexpr std::size_t N = detail::array_size::value; using return_type = xtensor_adaptor; @@ -268,7 +268,7 @@ namespace xt inline xtensor_adaptor::value, layout_type::dynamic> adapt(C&& container, SC&& shape, SS&& strides) { - static_assert(!std::is_integral>::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral>::value, "shape cannot be a integer"); constexpr std::size_t N = detail::array_size::value; using return_type = xtensor_adaptor, N, layout_type::dynamic>; return return_type(std::forward(container), @@ -314,7 +314,7 @@ namespace xt inline xtensor_adaptor, O, A>, detail::array_size::value, L> adapt(P&& pointer, typename A::size_type size, O ownership, const SC& shape, layout_type l = L, const A& alloc = A()) { - static_assert(!std::is_integral::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; constexpr std::size_t N = detail::array_size::value; @@ -340,7 +340,7 @@ namespace xt inline xtensor_adaptor, O, A>, detail::array_size::value, layout_type::dynamic> adapt(P&& pointer, typename A::size_type size, O ownership, SC&& shape, SS&& strides, const A& alloc = A()) { - static_assert(!std::is_integral>::value, "shape cannot be a integer"); + static_assert(!xtl::is_integral>::value, "shape cannot be a integer"); (void)ownership; using buffer_type = xbuffer_adaptor, O, A>; constexpr std::size_t N = detail::array_size::value; diff --git a/include/xtensor/xassign.hpp b/include/xtensor/xassign.hpp index b4f172b4d..e2de9a81f 100644 --- a/include/xtensor/xassign.hpp +++ b/include/xtensor/xassign.hpp @@ -530,9 +530,9 @@ namespace xt using argument_type = std::decay_t; using result_type = std::decay_t; - static const bool value = std::is_arithmetic::value && + static const bool value = xtl::is_arithmetic::value && (sizeof(result_type) < sizeof(argument_type) || - (std::is_integral::value && std::is_floating_point::value)); + (xtl::is_integral::value && std::is_floating_point::value)); }; template @@ -541,7 +541,7 @@ namespace xt using argument_type = std::decay_t; using result_type = std::decay_t; - static const bool value = std::is_signed::value != std::is_signed::value; + static const bool value = xtl::is_signed::value != xtl::is_signed::value; }; template diff --git a/include/xtensor/xbuilder.hpp b/include/xtensor/xbuilder.hpp index 56ece953f..113f05796 100644 --- a/include/xtensor/xbuilder.hpp +++ b/include/xtensor/xbuilder.hpp @@ -217,7 +217,7 @@ namespace xt // These methods should be private methods of arange_generator, however thi leads // to ICE on VS2015 - template )> + template )> inline void arange_assign_to(xexpression& e, U start, X step) noexcept { auto& de = e.derived_cast(); @@ -230,7 +230,7 @@ namespace xt } } - template >)> + template >)> inline void arange_assign_to(xexpression& e, U start, X step) noexcept { auto& buf = e.derived_cast().storage(); @@ -295,10 +295,10 @@ namespace xt }; template - using both_integer = xtl::conjunction, std::is_integral>; + using both_integer = xtl::conjunction, xtl::is_integral>; template - using integer_with_signed_integer = xtl::conjunction, std::is_signed>; + using integer_with_signed_integer = xtl::conjunction, xtl::is_signed>; template using integer_with_unsigned_integer = xtl::conjunction, std::is_unsigned>; diff --git a/include/xtensor/xcontainer.hpp b/include/xtensor/xcontainer.hpp index 87fc3682b..83b17cc4e 100644 --- a/include/xtensor/xcontainer.hpp +++ b/include/xtensor/xcontainer.hpp @@ -983,7 +983,7 @@ namespace xt template inline auto& xstrided_container::reshape(S&& shape, layout_type layout) & { - reshape_impl(std::forward(shape), std::is_signed::value_type>>(), std::forward(layout)); + reshape_impl(std::forward(shape), xtl::is_signed::value_type>>(), std::forward(layout)); return this->derived_cast(); } @@ -994,7 +994,7 @@ namespace xt using sh_type = rebind_container_t; sh_type sh = xtl::make_sequence(shape.size()); std::copy(shape.begin(), shape.end(), sh.begin()); - reshape_impl(std::move(sh), std::is_signed(), std::forward(layout)); + reshape_impl(std::move(sh), xtl::is_signed(), std::forward(layout)); return this->derived_cast(); } diff --git a/include/xtensor/xgenerator.hpp b/include/xtensor/xgenerator.hpp index 481c6c1a9..c02498b30 100644 --- a/include/xtensor/xgenerator.hpp +++ b/include/xtensor/xgenerator.hpp @@ -385,14 +385,14 @@ namespace xt template inline auto xgenerator::reshape(O&& shape) const & { - return reshape_view(*this, compute_shape(shape, std::is_signed::value_type>())); + return reshape_view(*this, compute_shape(shape, xtl::is_signed::value_type>())); } template template inline auto xgenerator::reshape(O&& shape) && { - return reshape_view(std::move(*this), compute_shape(shape, std::is_signed::value_type>())); + return reshape_view(std::move(*this), compute_shape(shape, xtl::is_signed::value_type>())); } template @@ -454,7 +454,7 @@ namespace xt using sh_type = xt::dynamic_shape; sh_type sh = xtl::make_sequence(shape.size()); std::copy(shape.begin(), shape.end(), sh.begin()); - return compute_shape(std::move(sh), std::is_signed()); + return compute_shape(std::move(sh), xtl::is_signed()); } template diff --git a/include/xtensor/xhistogram.hpp b/include/xtensor/xhistogram.hpp index 0228d26d2..1563db26c 100644 --- a/include/xtensor/xhistogram.hpp +++ b/include/xtensor/xhistogram.hpp @@ -237,7 +237,7 @@ namespace xt case histogram_algorithm::logspace: { using rhs_value_type - = std::conditional_t::value, double, value_type>; + = std::conditional_t::value, double, value_type>; xtensor bin_edges = xt::cast( xt::logspace(std::log10(left), std::log10(right), bins + 1)); @@ -405,7 +405,7 @@ namespace xt using input_value_type = typename std::decay_t::value_type; using size_type = typename std::decay_t::size_type; - static_assert(std::is_integral::value_type>::value, + static_assert(xtl::is_integral::value_type>::value, "Bincount data has to be integral type."); XTENSOR_ASSERT(data.dimension() == 1); XTENSOR_ASSERT(weights.dimension() == 1); diff --git a/include/xtensor/xio.hpp b/include/xtensor/xio.hpp index 02d9829fb..1c49326ac 100644 --- a/include/xtensor/xio.hpp +++ b/include/xtensor/xio.hpp @@ -428,7 +428,7 @@ namespace xt }; template - struct printer::value && !std::is_same::value>> + struct printer::value && !std::is_same::value>> { using value_type = std::decay_t; using cache_type = std::vector; @@ -460,7 +460,7 @@ namespace xt { m_max = math::abs(val); } - if (std::is_signed::value && val < 0) + if (xtl::is_signed::value && val < 0) { m_sign = true; } @@ -597,7 +597,7 @@ namespace xt }; template - struct printer::value && !xtl::is_complex::value>> + struct printer::value && !xtl::is_complex::value>> { using const_reference = typename T::const_reference; using value_type = std::decay_t; diff --git a/include/xtensor/xmanipulation.hpp b/include/xtensor/xmanipulation.hpp index 3cc5deb09..ebe95f520 100644 --- a/include/xtensor/xmanipulation.hpp +++ b/include/xtensor/xmanipulation.hpp @@ -49,7 +49,7 @@ namespace xt template auto squeeze(E&& e); - template ::value, int> = 0> + template ::value, int> = 0> auto squeeze(E&& e, S&& axis, Tag check_policy = Tag()); template @@ -442,7 +442,7 @@ namespace xt * @param check_policy select check_policy. With check_policy::full(), selecting an axis * which is greater than one will throw a runtime_error. */ - template ::value, int>> + template ::value, int>> inline auto squeeze(E&& e, S&& axis, Tag check_policy) { return detail::squeeze_impl(std::forward(e), std::forward(axis), check_policy); diff --git a/include/xtensor/xmath.hpp b/include/xtensor/xmath.hpp index f4a3f5e84..eca4eff69 100644 --- a/include/xtensor/xmath.hpp +++ b/include/xtensor/xmath.hpp @@ -223,21 +223,21 @@ XTENSOR_INT_SPECIALIZATION_IMPL(FUNC_NAME, RETURN_VAL, unsigned long long); // might return int instead of bool and the SIMD detection requires // bool return type. template - inline std::enable_if_t::value, bool> + inline std::enable_if_t::value, bool> isinf(const T& t) { return bool(std::isinf(t)); } template - inline std::enable_if_t::value, bool> + inline std::enable_if_t::value, bool> isnan(const T& t) { return bool(std::isnan(t)); } template - inline std::enable_if_t::value, bool> + inline std::enable_if_t::value, bool> isfinite(const T& t) { return bool(std::isfinite(t)); @@ -367,7 +367,7 @@ namespace detail { #define XTENSOR_REDUCER_FUNCTION(NAME, FUNCTOR, RESULT_TYPE, INIT) \ template >, xtl::negation>)> \ + XTL_REQUIRES(xtl::negation>, xtl::negation>)> \ inline auto NAME(E&& e, X&& axes, EVS es = EVS()) \ { \ using result_type = std::conditional_t::value, RESULT_TYPE, T>; \ @@ -380,7 +380,7 @@ namespace detail { } \ \ template >, std::is_integral)> \ + XTL_REQUIRES(xtl::negation>, xtl::is_integral)> \ inline auto NAME(E&& e, X axis, EVS es = EVS()) \ { \ return NAME(std::forward(e), {axis}, es); \ @@ -621,7 +621,7 @@ namespace detail { struct deg2rad { - template ::value, int> = 0> + template ::value, int> = 0> constexpr double operator()(const A& a) const noexcept { return a * xt::numeric_constants::PI / 180.0; @@ -633,7 +633,7 @@ namespace detail { return a * xt::numeric_constants::PI / A(180.0); } - template ::value, int> = 0> + template ::value, int> = 0> constexpr double simd_apply(const A& a) const noexcept { return a * xt::numeric_constants::PI / 180.0; @@ -648,7 +648,7 @@ namespace detail { struct rad2deg { - template ::value, int> = 0> + template ::value, int> = 0> constexpr double operator()(const A& a) const noexcept { return a * 180.0 / xt::numeric_constants::PI; @@ -660,7 +660,7 @@ namespace detail { return a * A(180.0) / xt::numeric_constants::PI; } - template ::value, int> = 0> + template ::value, int> = 0> constexpr double simd_apply(const A& a) const noexcept { return a * 180.0 / xt::numeric_constants::PI; @@ -835,7 +835,7 @@ namespace detail { struct sign_impl { template - static constexpr std::enable_if_t::value, T> run(T x) + static constexpr std::enable_if_t::value, T> run(T x) { return std::isnan(x) ? std::numeric_limits::quiet_NaN() : x == 0 ? T(copysign(T(0), x)) : T(copysign(T(1), x)); } @@ -1907,7 +1907,7 @@ namespace detail { } template >, std::is_integral)> + XTL_REQUIRES(xtl::negation>, xtl::is_integral)> inline auto mean(E&& e, X&& axes, D const& ddof, EVS es) { // sum cannot always be a double. It could be a complex number which cannot operate on @@ -1942,7 +1942,7 @@ namespace detail { #endif template , std::is_integral)> + XTL_REQUIRES(is_reducer_options, xtl::is_integral)> inline auto mean_noaxis(E&& e, const D& ddof, EVS es) { using value_type = typename std::conditional_t::value, double, T>; @@ -2004,7 +2004,7 @@ namespace detail { * @sa mean */ template , xtl::negation>)> + XTL_REQUIRES(is_reducer_options, xtl::negation>)> inline auto average(E&& e, W&& weights, X&& axes, EVS ev = EVS()) { xindex_type_t::shape_type> broadcast_shape; @@ -2091,7 +2091,7 @@ namespace detail { } template , std::is_integral)> + XTL_REQUIRES(is_reducer_options, xtl::is_integral)> inline auto variance(E&& e, D const& ddof, EVS es = EVS()) { auto cached_mean = mean(e, es)(); @@ -2134,7 +2134,7 @@ namespace detail { * @sa stddev, mean */ template >, std::is_integral)> + XTL_REQUIRES(xtl::negation>, xtl::is_integral)> inline auto variance(E&& e, X&& axes, const D& ddof, EVS es = EVS()) { decltype(auto) sc = detail::shared_forward(e); @@ -2155,7 +2155,7 @@ namespace detail { } template >, xtl::negation>>, is_reducer_options)> + XTL_REQUIRES(xtl::negation>, xtl::negation>>, is_reducer_options)> inline auto variance(E&& e, X&& axes, EVS es = EVS()) { return variance(std::forward(e), std::forward(axes), 0u, es); @@ -2522,7 +2522,7 @@ namespace detail { } template >, xtl::negation>)> + XTL_REQUIRES(xtl::negation>, xtl::negation>)> inline auto count_nonzero(E&& e, X&& axes, EVS es = EVS()) { COUNT_NON_ZEROS_CONTENT; @@ -2531,7 +2531,7 @@ namespace detail { } template >, std::is_integral)> + XTL_REQUIRES(xtl::negation>, xtl::is_integral)> inline auto count_nonzero(E&& e, X axis, EVS es = EVS()) { return count_nonzero(std::forward(e), {axis}, es); @@ -2565,14 +2565,14 @@ namespace detail { } template >, xtl::negation>)> + XTL_REQUIRES(xtl::negation>, xtl::negation>)> inline auto count_nonnan(E&& e, X&& axes, EVS es = EVS()) { return xt::count_nonzero(!xt::isnan(std::forward(e)), std::forward(axes), es); } template >, std::is_integral)> + XTL_REQUIRES(xtl::negation>, xtl::is_integral)> inline auto count_nonnan(E&& e, X&& axes, EVS es = EVS()) { return xt::count_nonzero(!xt::isnan(std::forward(e)), {axes}, es); diff --git a/include/xtensor/xnorm.hpp b/include/xtensor/xnorm.hpp index 0d7b76494..1e2c3b62f 100644 --- a/include/xtensor/xnorm.hpp +++ b/include/xtensor/xnorm.hpp @@ -37,7 +37,7 @@ namespace xt namespace traits_detail { - template ::value> + template ::value> struct norm_of_scalar_impl; template @@ -56,7 +56,7 @@ namespace xt using squared_norm_type = xtl::promote_type_t; }; - template ::value, + template ::value, bool floating = std::is_floating_point::value> struct norm_of_array_elements_impl; diff --git a/include/xtensor/xpad.hpp b/include/xtensor/xpad.hpp index 0fa254dec..263a8c8fe 100644 --- a/include/xtensor/xpad.hpp +++ b/include/xtensor/xpad.hpp @@ -293,7 +293,7 @@ namespace xt return detail::tile(std::forward(e), std::vector{reps}); } - template >)> + template >)> inline auto tile(E&& e, const C& reps) { return detail::tile(std::forward(e), reps); @@ -306,7 +306,7 @@ namespace xt * @param reps The number of repetitions of A along the first axis. * @return The tiled array. */ - template ::size_type, XTL_REQUIRES(std::is_integral)> + template ::size_type, XTL_REQUIRES(xtl::is_integral)> inline auto tile(E&& e, S reps) { std::vector tw(e.shape().size(), static_cast(1)); diff --git a/include/xtensor/xrandom.hpp b/include/xtensor/xrandom.hpp index 243992e34..4f3818586 100644 --- a/include/xtensor/xrandom.hpp +++ b/include/xtensor/xrandom.hpp @@ -239,7 +239,7 @@ namespace xt void shuffle(xexpression& e, E& engine = random::get_default_random_engine()); template - std::enable_if_t::value, xtensor> + std::enable_if_t::value, xtensor> permutation(T e, E& engine = random::get_default_random_engine()); template @@ -902,7 +902,7 @@ namespace xt * @return randomly permuted copy of container or arange. */ template - std::enable_if_t::value, xtensor> + std::enable_if_t::value, xtensor> permutation(T e, E& engine) { xt::xtensor res = xt::arange(e); diff --git a/include/xtensor/xslice.hpp b/include/xtensor/xslice.hpp index 17eb55fa7..20d32cfc9 100644 --- a/include/xtensor/xslice.hpp +++ b/include/xtensor/xslice.hpp @@ -368,11 +368,11 @@ namespace xt namespace detail { template - using disable_integral_keep = std::enable_if_t>::value, + using disable_integral_keep = std::enable_if_t>::value, xkeep_slice::value_type>>; template - using enable_integral_keep = std::enable_if_t::value, xkeep_slice>; + using enable_integral_keep = std::enable_if_t::value, xkeep_slice>; } /** @@ -492,11 +492,11 @@ namespace xt namespace detail { template - using disable_integral_drop = std::enable_if_t>::value, + using disable_integral_drop = std::enable_if_t>::value, xdrop_slice::value_type>>; template - using enable_integral_drop = std::enable_if_t::value, xdrop_slice>; + using enable_integral_drop = std::enable_if_t::value, xdrop_slice>; } /** @@ -549,9 +549,9 @@ namespace xt } template - inline std::enable_if_t::value && - std::is_integral::value && - std::is_integral::value, + inline std::enable_if_t::value && + xtl::is_integral::value && + xtl::is_integral::value, xstepped_range> get(std::size_t size) const { @@ -559,9 +559,9 @@ namespace xt } template - inline std::enable_if_t::value && - std::is_integral::value && - std::is_integral::value, + inline std::enable_if_t::value && + xtl::is_integral::value && + xtl::is_integral::value, xstepped_range> get(std::size_t size) const { @@ -569,9 +569,9 @@ namespace xt } template - inline std::enable_if_t::value && - !std::is_integral::value && - std::is_integral::value, + inline std::enable_if_t::value && + !xtl::is_integral::value && + xtl::is_integral::value, xstepped_range> get(std::size_t size) const { @@ -580,9 +580,9 @@ namespace xt } template - inline std::enable_if_t::value && - std::is_integral::value && - !std::is_integral::value, + inline std::enable_if_t::value && + xtl::is_integral::value && + !xtl::is_integral::value, xrange> get(std::size_t size) const { @@ -590,9 +590,9 @@ namespace xt } template - inline std::enable_if_t::value && - !std::is_integral::value && - std::is_integral::value, + inline std::enable_if_t::value && + !xtl::is_integral::value && + xtl::is_integral::value, xstepped_range> get(std::size_t size) const { @@ -602,9 +602,9 @@ namespace xt } template - inline std::enable_if_t::value && - !std::is_integral::value && - !std::is_integral::value, + inline std::enable_if_t::value && + !xtl::is_integral::value && + !xtl::is_integral::value, xrange> get(std::size_t size) const { @@ -612,9 +612,9 @@ namespace xt } template - inline std::enable_if_t::value && - std::is_integral::value && - !std::is_integral::value, + inline std::enable_if_t::value && + xtl::is_integral::value && + !xtl::is_integral::value, xrange> get(std::size_t size) const { @@ -622,9 +622,9 @@ namespace xt } template - inline std::enable_if_t::value && - !std::is_integral::value && - !std::is_integral::value, + inline std::enable_if_t::value && + !xtl::is_integral::value && + !xtl::is_integral::value, xall> get(std::size_t size) const { @@ -766,7 +766,7 @@ namespace xt }; template - struct cast_if_integer::value>> + struct cast_if_integer::value>> { using type = std::ptrdiff_t; @@ -892,7 +892,7 @@ namespace xt template inline decltype(auto) operator()(E& e, SL&& slice, std::size_t index) const { - return get_slice(e, std::forward(slice), index, std::is_signed>()); + return get_slice(e, std::forward(slice), index, xtl::is_signed>()); } private: diff --git a/include/xtensor/xsort.hpp b/include/xtensor/xsort.hpp index a5de83b68..1ed7fe990 100644 --- a/include/xtensor/xsort.hpp +++ b/include/xtensor/xsort.hpp @@ -406,7 +406,7 @@ namespace xt * @return partially sorted xcontainer */ template , - class = std::enable_if_t::value, int>> + class = std::enable_if_t::value, int>> inline R partition(const xexpression& e, const C& kth_container, placeholders::xtuph /*ax*/) { const auto& de = e.derived_cast(); @@ -451,7 +451,7 @@ namespace xt return partition(e, std::array({kth}), tag); } - template ::value, int>> + template ::value, int>> inline auto partition(const xexpression& e, const C& kth_container, std::ptrdiff_t axis = -1) { using eval_type = typename detail::sort_eval_type::type; @@ -553,7 +553,7 @@ namespace xt */ template ::type>::type, - class = std::enable_if_t::value, int>> + class = std::enable_if_t::value, int>> inline R argpartition(const xexpression& e, const C& kth_container, placeholders::xtuph) { using eval_type = typename detail::sort_eval_type::type; @@ -653,7 +653,7 @@ namespace xt } } - template ::value, int>> + template ::value, int>> inline auto argpartition(const xexpression& e, const C& kth_container, std::ptrdiff_t axis = -1) { using eval_type = typename detail::sort_eval_type::type; diff --git a/include/xtensor/xstrides.hpp b/include/xtensor/xstrides.hpp index a7de034ba..4b492b5a6 100644 --- a/include/xtensor/xstrides.hpp +++ b/include/xtensor/xstrides.hpp @@ -164,7 +164,7 @@ namespace xt template inline std::size_t compute_size(const shape_type& shape) noexcept { - return detail::compute_size_impl(shape, std::is_signed::value_type>>()); + return detail::compute_size_impl(shape, xtl::is_signed::value_type>>()); } namespace detail diff --git a/include/xtensor/xutils.hpp b/include/xtensor/xutils.hpp index 3fb4e59b5..85d8d1826 100644 --- a/include/xtensor/xutils.hpp +++ b/include/xtensor/xutils.hpp @@ -107,7 +107,7 @@ namespace xt }; template - using disable_integral_t = std::enable_if_t::value, R>; + using disable_integral_t = std::enable_if_t::value, R>; /******************************** * meta identity implementation * @@ -365,8 +365,8 @@ namespace xt } template - inline std::enable_if_t>::value && - std::is_signed::value_type>::value, + inline std::enable_if_t>::value && + xtl::is_signed::value_type>::value, rebind_container_t>> normalize_axis(E& expr, C&& axes) { @@ -384,7 +384,7 @@ namespace xt } template - inline std::enable_if_t>::value && std::is_unsigned::value_type>::value, C&&> + inline std::enable_if_t>::value && std::is_unsigned::value_type>::value, C&&> normalize_axis(E& expr, C&& axes) { static_cast(expr); @@ -394,7 +394,7 @@ namespace xt template inline auto forward_normalize(E& expr, C&& axes) - -> std::enable_if_t>::value, R> + -> std::enable_if_t>::value, R> { R res; xt::resize_container(res, xtl::sequence_size(axes)); @@ -410,7 +410,7 @@ namespace xt template inline auto forward_normalize(E& expr, C&& axes) - -> std::enable_if_t>::value && !std::is_same>::value, R> + -> std::enable_if_t>::value && !std::is_same>::value, R> { static_cast(expr); @@ -423,7 +423,7 @@ namespace xt template inline auto forward_normalize(E& expr, C&& axes) - -> std::enable_if_t>::value && std::is_same>::value, R&&> + -> std::enable_if_t>::value && std::is_same>::value, R&&> { static_cast(expr); XTENSOR_ASSERT(std::all_of(std::begin(axes), std::end(axes), [&expr](auto ax_el) { return ax_el < expr.dimension(); })); diff --git a/include/xtensor/xview.hpp b/include/xtensor/xview.hpp index 3aee09fc9..5d7ecd7d8 100644 --- a/include/xtensor/xview.hpp +++ b/include/xtensor/xview.hpp @@ -178,7 +178,7 @@ namespace xt { using slice = xtl::mpl::front_t; static constexpr bool is_range_slice = is_xrange::value; - static constexpr bool is_int_slice = std::is_integral::value; + static constexpr bool is_int_slice = xtl::is_integral::value; static constexpr bool is_all_slice = is_xall_slice::value; static constexpr bool have_all_seen = all_seen || is_all_slice; static constexpr bool have_range_seen = is_range_slice; @@ -206,7 +206,7 @@ namespace xt { using slice = xtl::mpl::front_t; static constexpr bool is_range_slice = is_xrange::value; - static constexpr bool is_int_slice = std::is_integral::value; + static constexpr bool is_int_slice = xtl::is_integral::value; static constexpr bool is_all_slice = is_xall_slice::value; static constexpr bool have_int_seen = int_seen || is_int_slice; diff --git a/include/xtensor/xview_utils.hpp b/include/xtensor/xview_utils.hpp index 79bdef881..727b9e17f 100644 --- a/include/xtensor/xview_utils.hpp +++ b/include/xtensor/xview_utils.hpp @@ -101,7 +101,7 @@ namespace xt { static constexpr std::size_t count(std::size_t i) noexcept { - return i ? (integral_count_impl::count(i - 1) + (std::is_integral>::value ? 1 : 0)) : 0; + return i ? (integral_count_impl::count(i - 1) + (xtl::is_integral>::value ? 1 : 0)) : 0; } }; @@ -194,7 +194,7 @@ namespace xt static constexpr std::size_t count_impl(std::size_t i) noexcept { return 1 + ( - std::is_integral>::value ? + xtl::is_integral>::value ? integral_skip_impl::count(i) : integral_skip_impl::count(i - 1) ); @@ -202,7 +202,7 @@ namespace xt static constexpr std::size_t count_impl() noexcept { - return std::is_integral>::value ? 1 + integral_skip_impl::count(0) : 0; + return xtl::is_integral>::value ? 1 + integral_skip_impl::count(0) : 0; } }; From 8066845e1bb1b5010a3fd31604bec0a244eb6373 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Thu, 15 Oct 2020 00:45:20 +0200 Subject: [PATCH 129/606] Fixed load_simd for xcomplex --- include/xtensor/xcontainer.hpp | 5 +++-- test/CMakeLists.txt | 10 ++++++++-- test/test_xcomplex.cpp | 10 ++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/include/xtensor/xcontainer.hpp b/include/xtensor/xcontainer.hpp index 87fc3682b..5431ab993 100644 --- a/include/xtensor/xcontainer.hpp +++ b/include/xtensor/xcontainer.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -707,7 +708,7 @@ namespace xt inline void xcontainer::store_simd(size_type i, const simd& e) { using align_mode = driven_align_mode_t; - xt_simd::store_simd(&(storage()[i]), e, align_mode()); + xt_simd::store_simd(std::addressof(storage()[i]), e, align_mode()); } template @@ -717,7 +718,7 @@ namespace xt //-> simd_return_type { using align_mode = driven_align_mode_t; - return xt_simd::load_simd(&(storage()[i]), align_mode()); + return xt_simd::load_simd(std::addressof(storage()[i]), align_mode()); } template diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dd742a558..06432b311 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -272,7 +272,10 @@ foreach(filename IN LISTS COMMON_BASE XTENSOR_TESTS) string(REPLACE ".cpp" "" targetname ${filename}) add_executable(${targetname} ${filename} ${TEST_HEADERS} ${XTENSOR_HEADERS}) if(XTENSOR_USE_XSIMD) - target_compile_definitions(${targetname} PRIVATE XTENSOR_USE_XSIMD) + target_compile_definitions(${targetname} + PRIVATE + XTENSOR_USE_XSIMD + XSIMD_ENABLE_XTL_COMPLEX) target_link_libraries(${targetname} PRIVATE xsimd) endif() if(XTENSOR_USE_TBB) @@ -296,7 +299,10 @@ endforeach() add_executable(test_xtensor_lib ${COMMON_BASE} ${XTENSOR_TESTS} ${TEST_HEADERS} ${XTENSOR_HEADERS}) if(XTENSOR_USE_XSIMD) - target_compile_definitions(test_xtensor_lib PRIVATE XTENSOR_USE_XSIMD) + target_compile_definitions(test_xtensor_lib + PRIVATE + XTENSOR_USE_XSIMD + XSIMD_ENABLE_XTL_COMPLEX) target_link_libraries(test_xtensor_lib PRIVATE xsimd) endif() if(XTENSOR_USE_TBB) diff --git a/test/test_xcomplex.cpp b/test/test_xcomplex.cpp index e02f5c350..d693f5cb5 100644 --- a/test/test_xcomplex.cpp +++ b/test/test_xcomplex.cpp @@ -10,6 +10,8 @@ #include "gtest/gtest.h" #include +#include + #include "xtensor/xarray.hpp" #include "xtensor/xbuilder.hpp" #include "xtensor/xcomplex.hpp" @@ -306,4 +308,12 @@ namespace xt EXPECT_EQ(rc(1).real(), r(1)); EXPECT_EQ(rc(2).real(), r(2)); } + + TEST(xcomplex, xcomplex) + { + using complex_type = xtl::xcomplex; + xt::xarray a = xt::ones(std::vector(3,7)); + + auto simd_loaded = a.template load_simd::size>(0); + } } From 3d5aafa1e51be781878dbb7660b06ddf09079745 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 16 Oct 2020 00:14:51 +0200 Subject: [PATCH 130/606] Upgraded to xtl 0.6.20 --- .azure-pipelines/azure-pipelines-win.yml | 2 +- environment-dev.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 0637ae430..974495c94 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -37,7 +37,7 @@ jobs: conda install cmake==3.14.0 ^ ninja ^ nlohmann_json ^ - xtl==0.6.18 ^ + xtl==0.6.20 ^ xsimd==7.4.8 ^ python=3.6 conda list diff --git a/environment-dev.yml b/environment-dev.yml index b7bfd103e..8fe69eb58 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -3,6 +3,6 @@ channels: - conda-forge dependencies: - cmake - - xtl=0.6.18 + - xtl=0.6.20 - xsimd=7.4.8 - nlohmann_json From 9e5c0bf4acc8b31ee2870b4e36fbb7d9999c1a84 Mon Sep 17 00:00:00 2001 From: Mario Emmenlauer Date: Fri, 16 Oct 2020 14:03:46 +0200 Subject: [PATCH 131/606] xstorage.hpp: Renamed a shadowing variable inside a function --- include/xtensor/xstorage.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/xtensor/xstorage.hpp b/include/xtensor/xstorage.hpp index 5d9f8696e..92e418191 100644 --- a/include/xtensor/xstorage.hpp +++ b/include/xtensor/xstorage.hpp @@ -380,7 +380,7 @@ namespace xt inline void uvector::reserve(size_type /*new_cap*/) { } - + template inline auto uvector::capacity() const noexcept -> size_type { @@ -1640,8 +1640,8 @@ namespace xt template constexpr static auto get() { - using cast_type = std::array; - return std::get(cast_type{X...}); + using tmp_cast_type = std::array; + return std::get(tmp_cast_type{X...}); } XTENSOR_FIXED_SHAPE_CONSTEXPR operator cast_type() const @@ -1686,7 +1686,7 @@ namespace xt XTENSOR_FIXED_SHAPE_CONSTEXPR bool empty() const { - return sizeof...(X) == 0; + return sizeof...(X) == 0; } private: From ca1a90bdd886a1369e6f178732c04e2eeb4d5321 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 16 Oct 2020 15:58:44 +0200 Subject: [PATCH 132/606] Release 0.21.8 --- README.md | 3 +- docs/source/changelog.rst | 48 ++++++++++++++++++++++++++++++ environment.yml | 2 +- include/xtensor/xtensor_config.hpp | 2 +- 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4ede9d9cd..23e85afd0 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,8 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| -| master | ^0.6.18 | ^7.4.8 | +| master | ^0.6.20 | ^7.4.8 | +| 0.21.8 | ^0.6.20 | ^7.4.8 | | 0.21.7 | ^0.6.18 | ^7.4.8 | | 0.21.6 | ^0.6.18 | ^7.4.8 | | 0.21.5 | ^0.6.12 | ^7.4.6 | diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 88af028f8..c64c33e37 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -7,6 +7,54 @@ Changelog ========= +0.21.8 +------ + +- Fix undefined behavior while testing shifts + `#2175 `_ +- Fix ``zarray`` initialization from ``zarray`` + `#2180 `_ +- Portable and generic implementation of endianess detection + `#2182 `_ +- Fix xnpy save padding computation + `#2183 `_ +- Only use ``-march=native`` if it's available + `#2184 `_ +- Fix ``xchunked_array`` assignment + `#2177 `_ +- Add specific ``xchunked_array`` constructor for ``xchunk_store_manager`` + `#2188 `_ +- Make xnpy tests aware of both little and big endian targets + `#2189 `_ +- Fixed constructors of ``xchunked_array`` + `#2190 `_ +- First implementation of ``zchunked_wrapper`` + `#2193 `_ +- Don't mark dirty a resized or reshaped ``xfile_array`` + `#2194 `_ +- Replaced catch-all constructor of ``zarray`` with more restrictive ones + `#2195 `_ +- Fixed SFINAE based on ``xchunked_store_manager`` + `#2197 `_ +- Fix generated cmake config to include missing required lib + `#2200 `_ +- Add ``set_chunk_shape`` to the first chunk of the pool + `#2198 `_ +- Chunked array refactoring + `#2201 `_ +- Refactored ``xchunked_array`` semantic + `#2202 `_ +- Added missing header to CMakeLists.txt + `#2203 `_ +- Fixed ``load_simd`` for ``xcomplex`` + `#2204 `_ +- Upgraded to xtl 0.6.20 + `#2206 `_ +- changed std traits to new ``xtl::xtraits`` + `#2205 `_ +- ``xstorage.hpp``: Renamed a shadowing variable inside a function + `#2207 `_ + 0.21.7 ------ diff --git a/environment.yml b/environment.yml index 9cf73c357..e885eed95 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xtensor channels: - conda-forge dependencies: - - xtensor=0.21.7 + - xtensor=0.21.8 - xtensor-blas=0.17.2 - xeus-cling=0.8.1 - blas * *openblas" diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index 46206aaf9..0853f62dd 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -12,7 +12,7 @@ #define XTENSOR_VERSION_MAJOR 0 #define XTENSOR_VERSION_MINOR 21 -#define XTENSOR_VERSION_PATCH 7 +#define XTENSOR_VERSION_PATCH 8 // DETECT 3.6 <= clang < 3.8 for compiler bug workaround. #ifdef __clang__ From d509a41f5e7ca5fce5a9c3b795f7cf03ca8eb3ca Mon Sep 17 00:00:00 2001 From: Mario Emmenlauer Date: Mon, 19 Oct 2020 07:21:20 +0200 Subject: [PATCH 133/606] xcontainer.hpp: Renamed a shadowing type name inside a function (#2208) Renamed a shadowing types and names --- include/xtensor/xassign.hpp | 12 ++++++------ include/xtensor/xcontainer.hpp | 8 ++++---- include/xtensor/xgenerator.hpp | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/xtensor/xassign.hpp b/include/xtensor/xassign.hpp index e2de9a81f..546b277fc 100644 --- a/include/xtensor/xassign.hpp +++ b/include/xtensor/xassign.hpp @@ -334,7 +334,7 @@ namespace xt static constexpr bool is_bool_conversion() { return is_bool::value && !is_bool::value; } static constexpr bool contiguous_layout() { return E1::contiguous_layout && E2::contiguous_layout; } - static constexpr bool convertible_types() { return std::is_convertible::value + static constexpr bool convertible_types() { return std::is_convertible::value && !is_bool_conversion(); } static constexpr bool use_xsimd() { return xt_simd::simd_traits::size > 1; } @@ -345,7 +345,7 @@ namespace xt static constexpr bool simd_interface() { return has_simd_interface(); } public: - + // constexpr methods instead of constexpr data members avoid the need of definitions at namespace // scope of these data members (since they are odr-used). @@ -565,13 +565,13 @@ namespace xt template inline void stepper_assigner::run() { - using size_type = typename E1::size_type; + using tmp_size_type = typename E1::size_type; using argument_type = std::decay_t; using result_type = std::decay_t; constexpr bool needs_cast = has_assign_conversion::value; - size_type s = m_e1.size(); - for (size_type i = 0; i < s; ++i) + tmp_size_type s = m_e1.size(); + for (tmp_size_type i = 0; i < s; ++i) { *m_lhs = conditional_cast(*m_rhs); stepper_tools::increment_stepper(*this, m_index, m_e1.shape()); @@ -906,7 +906,7 @@ namespace xt std::size_t inner_loop_size, outer_loop_size, cut; std::tie(inner_loop_size, outer_loop_size, cut) = strided_assign_detail::get_loop_sizes(e1, e2, is_row_major); - if ((is_row_major && cut == e1.dimension()) || (!is_row_major && cut == 0)) + if ((is_row_major && cut == e1.dimension()) || (!is_row_major && cut == 0)) { return fallback_assigner(e1, e2).run(); } diff --git a/include/xtensor/xcontainer.hpp b/include/xtensor/xcontainer.hpp index ebf56a70c..d6d3c9b82 100644 --- a/include/xtensor/xcontainer.hpp +++ b/include/xtensor/xcontainer.hpp @@ -1026,14 +1026,14 @@ namespace xt template inline void xstrided_container::reshape_impl(S&& _shape, std::true_type /* is signed */, layout_type layout) { - using value_type = typename std::decay_t::value_type; + using tmp_value_type = typename std::decay_t::value_type; auto new_size = compute_size(_shape); if (this->size() % new_size) { XTENSOR_THROW(std::runtime_error, "Negative axis size cannot be inferred. Shape mismatch."); } std::decay_t shape = _shape; - value_type accumulator = 1; + tmp_value_type accumulator = 1; std::size_t neg_idx = 0; std::size_t i = 0; for(auto it = shape.begin(); it != shape.end(); ++it, i++) @@ -1048,7 +1048,7 @@ namespace xt } if(accumulator < 0) { - shape[neg_idx] = static_cast(this->size()) / std::abs(accumulator); + shape[neg_idx] = static_cast(this->size()) / std::abs(accumulator); } else if(this->size() != new_size) { @@ -1060,7 +1060,7 @@ namespace xt resize_container(m_backstrides, m_shape.size()); compute_strides(m_shape, m_layout, m_strides, m_backstrides); } - + template inline auto xstrided_container::mutable_layout() noexcept -> layout_type& { diff --git a/include/xtensor/xgenerator.hpp b/include/xtensor/xgenerator.hpp index c02498b30..d1c223487 100644 --- a/include/xtensor/xgenerator.hpp +++ b/include/xtensor/xgenerator.hpp @@ -467,14 +467,14 @@ namespace xt template inline void xgenerator::adapt_index(I& arg, Args&... args) const { - using value_type = typename decltype(m_shape)::value_type; + using tmp_value_type = typename decltype(m_shape)::value_type; if (sizeof...(Args) + 1 > m_shape.size()) { adapt_index(args...); } else { - if (static_cast(arg) >= m_shape[dim] && m_shape[dim] == 1) + if (static_cast(arg) >= m_shape[dim] && m_shape[dim] == 1) { arg = 0; } From c76e2936823a77a3beb7e5452e128cb6244d225c Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Mon, 19 Oct 2020 11:01:52 +0200 Subject: [PATCH 134/606] Adding macro XTENSOR_SELECT_ALIGN (#2152) Adding macro XTENSOR_SELECT_ALIGN --- docs/source/index.rst | 1 + docs/source/sfinae.rst | 2 +- docs/source/xsimd.rst | 52 ++++++++++++++++++++++++++++++ include/xtensor/xstorage.hpp | 5 +-- include/xtensor/xtensor_config.hpp | 4 +++ test/CMakeLists.txt | 1 + test/test_xsimd.cpp | 51 +++++++++++++++++++++++++++++ 7 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 docs/source/xsimd.rst create mode 100644 test/test_xsimd.cpp diff --git a/docs/source/index.rst b/docs/source/index.rst index f8556bc70..051df2b16 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -70,6 +70,7 @@ for details. histogram random sfinae + xsimd file_loading build-options pitfall diff --git a/docs/source/sfinae.rst b/docs/source/sfinae.rst index bd53f3644..30884451b 100644 --- a/docs/source/sfinae.rst +++ b/docs/source/sfinae.rst @@ -4,7 +4,7 @@ The full license is in the file LICENSE, distributed with this software. -.. _histogram: +.. _sfinae: SFINAE ====== diff --git a/docs/source/xsimd.rst b/docs/source/xsimd.rst new file mode 100644 index 000000000..f7de447c4 --- /dev/null +++ b/docs/source/xsimd.rst @@ -0,0 +1,52 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +.. _xsimd: + +xsimd +===== + +Alignment of fixed-size members +------------------------------- + +.. note:: + + If you are using ``C++ >= 17`` you should not have to worry about this. + +If you define a structure having members of fixed-size xtensor types, +you must ensure that the buffers properly aligned. +For this you can use the macro ``XTENSOR_SELECT_ALIGN`` available in +``xtensor/xtensor_config.hpp``. +Consider the following example: + +.. code-block:: cpp + + template + class alignas(XTENSOR_SELECT_ALIGN(T)) Foo + { + public: + + using allocator_type = std::conditional_t, + std::allocator>; + + Foo(T fac) : m_fac(fac) + { + m_bar.fill(fac); + } + + auto get() const + { + return m_bar; + } + + private: + + xt::xtensor_fixed> m_bar; + T m_fac; + }; + +Whereby it is important to store the fixed-sized xtensor type (in this case ``xt::xtensor_fixed>``) as first member. diff --git a/include/xtensor/xstorage.hpp b/include/xtensor/xstorage.hpp index 92e418191..686cb7176 100644 --- a/include/xtensor/xstorage.hpp +++ b/include/xtensor/xstorage.hpp @@ -1368,8 +1368,6 @@ namespace xt lhs.swap(rhs); } - #define XTENSOR_SELECT_ALIGN (XTENSOR_DEFAULT_ALIGNMENT != 0 ? XTENSOR_DEFAULT_ALIGNMENT : alignof(T)) - template struct rebind_container> { @@ -1383,7 +1381,7 @@ namespace xt * * To be moved to xtl, along with the rest of xstorage.hpp */ - template + template class alignas(Align) aligned_array : public std::array { public: @@ -1936,6 +1934,5 @@ namespace std #endif #undef XTENSOR_CONST -#undef XTENSOR_SELECT_ALIGN #endif diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index 0853f62dd..016786f83 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -110,6 +110,10 @@ #define XTENSOR_OPENMP_TRESHOLD 0 #endif +#ifndef XTENSOR_SELECT_ALIGN +#define XTENSOR_SELECT_ALIGN(T) (XTENSOR_DEFAULT_ALIGNMENT != 0 ? XTENSOR_DEFAULT_ALIGNMENT : alignof(T)) +#endif + #ifdef IN_DOXYGEN namespace xtl { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 06432b311..ad25520a8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -219,6 +219,7 @@ set(XTENSOR_TESTS test_xrandom.cpp test_xrepeat.cpp test_xsort.cpp + test_xsimd.cpp test_xvectorize.cpp test_extended_xmath_interp.cpp test_extended_broadcast_view.cpp diff --git a/test/test_xsimd.cpp b/test/test_xsimd.cpp new file mode 100644 index 000000000..902d5aa36 --- /dev/null +++ b/test/test_xsimd.cpp @@ -0,0 +1,51 @@ +/*************************************************************************** +* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include +#include + +#include "gtest/gtest.h" +#include "xtensor/xfixed.hpp" +#include "xtensor/xtensor_config.hpp" + +template +class alignas(XTENSOR_SELECT_ALIGN(T)) Foo +{ +public: + + using allocator_type = std::conditional_t, + std::allocator>; + + Foo(T fac) : m_fac(fac) + { + m_bar.fill(fac); + } + + auto get() const + { + return m_bar; + } + +private: + + xt::xtensor_fixed> m_bar; + T m_fac; +}; + +namespace xt +{ + + TEST(xsimd, alignas) + { + int fac = 10; + Foo foo(10); + EXPECT_TRUE(xt::sum(foo.get())() == fac * 10 * 10); + } +} From 497d66e6a662560fb77eb7357aad4ee4c6ce0943 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 19 Oct 2020 16:47:21 +0200 Subject: [PATCH 135/606] Add chunk_memory_layout to chunked_array factory --- include/xtensor/xchunked_array.hpp | 68 +++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 97f89d69d..29aebeafc 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -124,7 +124,7 @@ namespace xt static constexpr bool contiguous_layout = false; template - xchunked_array(chunk_storage_type&& chunks, S&& shape, S&& chunk_shape); + xchunked_array(chunk_storage_type&& chunks, S&& shape, S&& chunk_shape, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); ~xchunked_array() = default; xchunked_array(const xchunked_array&) = default; @@ -134,10 +134,10 @@ namespace xt xchunked_array& operator=(xchunked_array&&) = default; template - xchunked_array(const xexpression&e , chunk_storage_type&& chunks); + xchunked_array(const xexpression&e , chunk_storage_type&& chunks, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); template - xchunked_array(const xexpression& e, chunk_storage_type&& chunks, S&& chunk_shape); + xchunked_array(const xexpression& e, chunk_storage_type&& chunks, S&& chunk_shape, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); template xchunked_array& operator=(const xexpression& e); @@ -195,7 +195,7 @@ namespace xt using dynamic_indexes_type = std::pair, std::vector>; template - void resize(S1&& shape, S2&& chunk_shape); + void resize(S1&& shape, S2&& chunk_shape, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); template indexes_type get_indexes(Idxs... idxs) const; @@ -220,16 +220,16 @@ namespace xt template constexpr bool is_chunked(const xexpression& e); - template - xchunked_array>, EXT> chunked_array(S&& shape, S&& chunk_shape); + template + xchunked_array>, EXT> chunked_array(S&& shape, S&& chunk_shape, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); - template + template xchunked_array>, EXT> - chunked_array(const xexpression& e, S&& chunk_shape); + chunked_array(const xexpression& e, S&& chunk_shape, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); - template + template xchunked_array>, EXT> - chunked_array(const xexpression&e); + chunked_array(const xexpression&e, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); /******************************* * chunk_helper implementation * @@ -251,12 +251,12 @@ namespace xt } template - static void resize(E& chunks, const S1& container_shape, const S2& chunk_shape) + static void resize(E& chunks, const S1& container_shape, const S2& chunk_shape, layout_type chunk_memory_layout) { chunks.resize(container_shape); for(auto& c: chunks) { - c.resize(chunk_shape); + c.resize(chunk_shape, chunk_memory_layout); } } }; @@ -271,9 +271,9 @@ namespace xt } template - static void resize(E& chunks, const S1& container_shape, const S2& chunk_shape) + static void resize(E& chunks, const S1& container_shape, const S2& chunk_shape, layout_type chunk_memory_layout) { - chunks.resize(container_shape, chunk_shape); + chunks.resize(container_shape, chunk_shape, chunk_memory_layout); } }; @@ -288,27 +288,27 @@ namespace xt return return_type::value; } - template - inline xchunked_array>, EXT> chunked_array(S&& shape, S&& chunk_shape) + template + inline xchunked_array>, EXT> chunked_array(S&& shape, S&& chunk_shape, layout_type chunk_memory_layout) { - using chunk_storage = xarray>; - return xchunked_array(chunk_storage(), std::forward(shape), std::forward(chunk_shape)); + using chunk_storage = xarray>; + return xchunked_array(chunk_storage(), std::forward(shape), std::forward(chunk_shape), chunk_memory_layout); } - template + template inline xchunked_array>, EXT> - chunked_array(const xexpression& e, S&& chunk_shape) + chunked_array(const xexpression& e, S&& chunk_shape, layout_type chunk_memory_layout) { - using chunk_storage = xarray>; - return xchunked_array(e, chunk_storage(), std::forward(chunk_shape)); + using chunk_storage = xarray>; + return xchunked_array(e, chunk_storage(), std::forward(chunk_shape), chunk_memory_layout); } - template + template inline xchunked_array>, EXT> - chunked_array(const xexpression& e) + chunked_array(const xexpression& e, layout_type chunk_memory_layout) { - using chunk_storage = xarray>; - return xchunked_array(e, chunk_storage()); + using chunk_storage = xarray>; + return xchunked_array(e, chunk_storage(), chunk_memory_layout); } /************************************ @@ -421,25 +421,25 @@ namespace xt template template - inline xchunked_array::xchunked_array(CS&& chunks, S&& shape, S&& chunk_shape) + inline xchunked_array::xchunked_array(CS&& chunks, S&& shape, S&& chunk_shape, layout_type chunk_memory_layout) : m_chunks(std::move(chunks)) { - resize(std::forward(shape), std::forward(chunk_shape)); + resize(std::forward(shape), std::forward(chunk_shape), chunk_memory_layout); } template template - inline xchunked_array::xchunked_array(const xexpression& e, CS&& chunks) - : xchunked_array(e, std::move(chunks), detail::chunk_helper::chunk_shape(e)) + inline xchunked_array::xchunked_array(const xexpression& e, CS&& chunks, layout_type chunk_memory_layout) + : xchunked_array(e, std::move(chunks), detail::chunk_helper::chunk_shape(e), chunk_memory_layout) { } template template - inline xchunked_array::xchunked_array(const xexpression& e, CS&& chunks, S&& chunk_shape) + inline xchunked_array::xchunked_array(const xexpression& e, CS&& chunks, S&& chunk_shape, layout_type chunk_memory_layout) : m_chunks(std::move(chunks)) { - resize(e.derived_cast().shape(), std::forward(chunk_shape)); + resize(e.derived_cast().shape(), std::forward(chunk_shape), chunk_memory_layout); semantic_base::assign_xexpression(e); } @@ -570,7 +570,7 @@ namespace xt template template - inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape) + inline void xchunked_array::resize(S1&& shape, S2&& chunk_shape, layout_type chunk_memory_layout) { // compute chunk number in each dimension (shape_of_chunks) std::vector shape_of_chunks(shape.size()); @@ -588,7 +588,7 @@ namespace xt } ); - detail::chunk_helper::resize(m_chunks, shape_of_chunks, chunk_shape); + detail::chunk_helper::resize(m_chunks, shape_of_chunks, chunk_shape, chunk_memory_layout); m_shape = xtl::forward_sequence(shape); m_chunk_shape = xtl::forward_sequence(chunk_shape); From 5921553897efce220de931fea884bbc17b38c525 Mon Sep 17 00:00:00 2001 From: Mario Emmenlauer Date: Wed, 21 Oct 2020 14:16:00 +0200 Subject: [PATCH 136/606] test/CMakeLists.txt: Modernized GTest-integration --- CMakeLists.txt | 1 + test/CMakeLists.txt | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64dfd0755..469262311 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,6 +237,7 @@ if(MSVC AND DISABLE_MSVC_ITERATOR_CHECK) endif() if(BUILD_TESTS) + enable_testing() add_subdirectory(test) endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ad25520a8..fbbfe7b5d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -12,6 +12,8 @@ cmake_minimum_required(VERSION 3.1) if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) project(xtensor-test) + enable_testing() + find_package(xtensor REQUIRED CONFIG) set(XTENSOR_INCLUDE_DIR ${xtensor_INCLUDE_DIRS}) endif () @@ -137,7 +139,10 @@ if(DOWNLOAD_GTEST OR GTEST_SRC_DIR) ${CMAKE_CURRENT_BINARY_DIR}/googletest-build EXCLUDE_FROM_ALL) set(GTEST_INCLUDE_DIRS "${gtest_SOURCE_DIR}/include") - set(GTEST_BOTH_LIBRARIES gtest_main gtest) + add_library(GTest::GTest INTERFACE IMPORTED) + target_link_libraries(GTest::GTest INTERFACE gtest) + add_library(GTest::Main INTERFACE IMPORTED) + target_link_libraries(GTest::Main INTERFACE gtest_main) else() find_package(GTest REQUIRED) endif() @@ -291,11 +296,12 @@ foreach(filename IN LISTS COMMON_BASE XTENSOR_TESTS) add_dependencies(${targetname} gtest_main) endif() target_include_directories(${targetname} PRIVATE ${XTENSOR_INCLUDE_DIR}) - target_link_libraries(${targetname} PRIVATE xtensor ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(${targetname} PRIVATE xtensor GTest::GTest GTest::Main ${CMAKE_THREAD_LIBS_INIT}) add_custom_target( x${targetname} COMMAND ${targetname} DEPENDS ${targetname} ${filename} ${XTENSOR_HEADERS}) + add_test(NAME ${targetname} COMMAND ${targetname}) endforeach() add_executable(test_xtensor_lib ${COMMON_BASE} ${XTENSOR_TESTS} ${TEST_HEADERS} ${XTENSOR_HEADERS}) @@ -320,9 +326,10 @@ if(DOWNLOAD_GTEST OR GTEST_SRC_DIR) endif() target_include_directories(test_xtensor_lib PRIVATE ${XTENSOR_INCLUDE_DIR}) -target_link_libraries(test_xtensor_lib PRIVATE xtensor ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(test_xtensor_lib PRIVATE xtensor GTest::GTest GTest::Main ${CMAKE_THREAD_LIBS_INIT}) add_custom_target(xtest COMMAND test_xtensor_lib DEPENDS test_xtensor_lib) +add_test(NAME xtest COMMAND test_xtensor_lib) # Some files will be compiled twice, however compiling common files in a static # library and linking test_xtensor_lib with it removes half of the tests at @@ -334,5 +341,5 @@ if(DOWNLOAD_GTEST OR GTEST_SRC_DIR) add_dependencies(test_xtensor_core_lib gtest_main) endif() -target_link_libraries(test_xtensor_core_lib PRIVATE xtensor ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(test_xtensor_core_lib PRIVATE xtensor GTest::GTest GTest::Main ${CMAKE_THREAD_LIBS_INIT}) add_custom_target(coverity COMMAND coverity_scan DEPENDS test_xtensor_core_lib) From 1f6d76abe8f378734568024a7d3bd85f54449697 Mon Sep 17 00:00:00 2001 From: Gregory Lemercier Date: Fri, 23 Oct 2020 16:52:39 +0200 Subject: [PATCH 137/606] xnpy.hpp: fix multiple definition of 'host_endian_char' variable --- include/xtensor/xnpy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xnpy.hpp b/include/xtensor/xnpy.hpp index ba93289be..1d347f92f 100644 --- a/include/xtensor/xnpy.hpp +++ b/include/xtensor/xnpy.hpp @@ -60,7 +60,7 @@ namespace xt const char big_endian_char = '>'; const char no_endian_char = '|'; - char host_endian_char = (is_big_endian() ? big_endian_char : little_endian_char); + static char host_endian_char = (is_big_endian() ? big_endian_char : little_endian_char); template inline void write_magic(O& ostream, From c0cc47a8b3e43f618de4cec820c350e13bf0f36e Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 27 Oct 2020 10:03:35 +0100 Subject: [PATCH 138/606] Made global variable const to force internal linkage --- include/xtensor/xnpy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xnpy.hpp b/include/xtensor/xnpy.hpp index 1d347f92f..a3b12b93e 100644 --- a/include/xtensor/xnpy.hpp +++ b/include/xtensor/xnpy.hpp @@ -60,7 +60,7 @@ namespace xt const char big_endian_char = '>'; const char no_endian_char = '|'; - static char host_endian_char = (is_big_endian() ? big_endian_char : little_endian_char); + const char host_endian_char = (is_big_endian() ? big_endian_char : little_endian_char); template inline void write_magic(O& ostream, From 20a8a89650d609e1ce6d874e9496a4aee01c367d Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Thu, 29 Oct 2020 20:52:55 +0100 Subject: [PATCH 139/606] Use xtl::endianness instead of bundling it --- include/xtensor/xnpy.hpp | 36 ++++++++++++++++++------------------ test/test_xnpy.cpp | 13 ++++++++++--- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/include/xtensor/xnpy.hpp b/include/xtensor/xnpy.hpp index a3b12b93e..cb4ec4fe4 100644 --- a/include/xtensor/xnpy.hpp +++ b/include/xtensor/xnpy.hpp @@ -15,6 +15,7 @@ // relicensed from MIT License with permission #include +#include #include #include @@ -43,24 +44,9 @@ namespace xt namespace detail { - /* Test for endianess. Compiler can optimize that to a single constant. */ - static inline bool is_big_endian() - { - uint32_t utmp = 0x01020304; - char btmp[sizeof(utmp)]; - std::memcpy(&btmp[0], &utmp, sizeof(utmp)); - const bool big_endian = btmp[0] == 0x01; - return big_endian; - } - const char magic_string[] = "\x93NUMPY"; - const std::size_t magic_string_length = 6; - - const char little_endian_char = '<'; - const char big_endian_char = '>'; - const char no_endian_char = '|'; + const std::size_t magic_string_length = sizeof(magic_string) - 1; - const char host_endian_char = (is_big_endian() ? big_endian_char : little_endian_char); template inline void write_magic(O& ostream, @@ -126,9 +112,23 @@ namespace xt } template - constexpr char get_endianess() + inline char get_endianess() { - return sizeof(T) <= sizeof(char) ? no_endian_char : host_endian_char; + constexpr char little_endian_char = '<'; + constexpr char big_endian_char = '>'; + constexpr char no_endian_char = '|'; + + if(sizeof(T) <= sizeof(char)) + return no_endian_char; + + switch(xtl::endianness()) { + case xtl::endian::little_endian: + return little_endian_char; + case xtl::endian::big_endian: + return big_endian_char; + default: + return no_endian_char; + } } template diff --git a/test/test_xnpy.cpp b/test/test_xnpy.cpp index 49df9fb32..c4b311fb8 100644 --- a/test/test_xnpy.cpp +++ b/test/test_xnpy.cpp @@ -18,10 +18,17 @@ namespace xt { std::string get_load_filename(std::string const& npy_prefix, layout_type lt = layout_type::row_major) { - using detail::is_big_endian; std::string lts = lt == layout_type::row_major ? "" : "_fortran"; - std::string endianess = is_big_endian() ? ".be" : ".le"; - return npy_prefix + lts + endianess + ".npy"; + std::string endianness; + switch(xtl::endianness()) { + case xtl::endian::big_endian: + endianness = ".be"; + break; + case xtl::endian::little_endian: + endianness = ".le"; + break; + }; + return npy_prefix + lts + endianness + ".npy"; } TEST(xnpy, load) From 00fa064315bde568e3ff30603ac119e8fd8e3090 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Fri, 30 Oct 2020 00:57:49 +0100 Subject: [PATCH 140/606] Upgraded to xtl 0.6.21 --- .azure-pipelines/azure-pipelines-win.yml | 2 +- environment-dev.yml | 2 +- include/xtensor/xnpy.hpp | 13 ++++++++----- test/test_xnpy.cpp | 22 ++++++++++++---------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 974495c94..d91b94660 100644 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -37,7 +37,7 @@ jobs: conda install cmake==3.14.0 ^ ninja ^ nlohmann_json ^ - xtl==0.6.20 ^ + xtl==0.6.21 ^ xsimd==7.4.8 ^ python=3.6 conda list diff --git a/environment-dev.yml b/environment-dev.yml index 8fe69eb58..ac654ea91 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -3,6 +3,6 @@ channels: - conda-forge dependencies: - cmake - - xtl=0.6.20 + - xtl=0.6.21 - xsimd=7.4.8 - nlohmann_json diff --git a/include/xtensor/xnpy.hpp b/include/xtensor/xnpy.hpp index cb4ec4fe4..b61cb6226 100644 --- a/include/xtensor/xnpy.hpp +++ b/include/xtensor/xnpy.hpp @@ -119,14 +119,17 @@ namespace xt constexpr char no_endian_char = '|'; if(sizeof(T) <= sizeof(char)) - return no_endian_char; + { + return no_endian_char; + } - switch(xtl::endianness()) { - case xtl::endian::little_endian: + switch(xtl::endianness()) + { + case xtl::endian::little_endian: return little_endian_char; - case xtl::endian::big_endian: + case xtl::endian::big_endian: return big_endian_char; - default: + default: return no_endian_char; } } diff --git a/test/test_xnpy.cpp b/test/test_xnpy.cpp index c4b311fb8..13b50e632 100644 --- a/test/test_xnpy.cpp +++ b/test/test_xnpy.cpp @@ -17,18 +17,20 @@ namespace xt { - std::string get_load_filename(std::string const& npy_prefix, layout_type lt = layout_type::row_major) { - std::string lts = lt == layout_type::row_major ? "" : "_fortran"; - std::string endianness; - switch(xtl::endianness()) { + std::string get_load_filename(std::string const& npy_prefix, layout_type lt = layout_type::row_major) + { + std::string lts = lt == layout_type::row_major ? "" : "_fortran"; + std::string endianness; + switch(xtl::endianness()) + { case xtl::endian::big_endian: - endianness = ".be"; - break; + endianness = ".be"; + break; case xtl::endian::little_endian: - endianness = ".le"; - break; - }; - return npy_prefix + lts + endianness + ".npy"; + endianness = ".le"; + break; + } + return npy_prefix + lts + endianness + ".npy"; } TEST(xnpy, load) From f6a346b244875299cb17fc7adf85a516fef143c9 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 3 Nov 2020 17:31:01 +0100 Subject: [PATCH 141/606] Fix call to resize of chunk container --- include/xtensor/xchunked_array.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 29aebeafc..65ce130ca 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -273,7 +273,7 @@ namespace xt template static void resize(E& chunks, const S1& container_shape, const S2& chunk_shape, layout_type chunk_memory_layout) { - chunks.resize(container_shape, chunk_shape, chunk_memory_layout); + chunks.resize(container_shape); } }; From a377fb84c29624e91b6b2451018f3aa3abec1c4f Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 4 Nov 2020 22:41:24 +0100 Subject: [PATCH 142/606] Release 0.21.9 --- README.md | 3 ++- docs/source/changelog.rst | 20 ++++++++++++++++++++ environment.yml | 2 +- include/xtensor/xtensor_config.hpp | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 23e85afd0..cd7904879 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,8 @@ library: | `xtensor` | `xtl` |`xsimd` (optional) | |-----------|---------|-------------------| -| master | ^0.6.20 | ^7.4.8 | +| master | ^0.6.21 | ^7.4.8 | +| 0.21.9 | ^0.6.21 | ^7.4.8 | | 0.21.8 | ^0.6.20 | ^7.4.8 | | 0.21.7 | ^0.6.18 | ^7.4.8 | | 0.21.6 | ^0.6.18 | ^7.4.8 | diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index c64c33e37..5cd5db7cf 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -7,6 +7,26 @@ Changelog ========= +0.21.9 +------ + +- Adding macro ``XTENSOR_SELECT_ALIGN`` + `#2152 `_ +- xcontainer.hpp: Renamed a shadowing type name inside a function + `#2208 `_ +- Add chunk_memory_layout to chunked_array factory + `#2211 `_ +- CMake: Modernized GTest-integration + `#2212 `_ +- ``xnpy.hpp``: fix multiple definition of 'host_endian_char' variable when included in different linked objects + `#2214 `_ +- Made global variable const to force internal linkage + `#2216 `_ +- Use xtl::endianness instead of bundling it + `#2218 `_ +- Fix call to resize of chunk container + `#2219 `_ + 0.21.8 ------ diff --git a/environment.yml b/environment.yml index e885eed95..c0665ba7b 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xtensor channels: - conda-forge dependencies: - - xtensor=0.21.8 + - xtensor=0.21.9 - xtensor-blas=0.17.2 - xeus-cling=0.8.1 - blas * *openblas" diff --git a/include/xtensor/xtensor_config.hpp b/include/xtensor/xtensor_config.hpp index 016786f83..61bcef2ea 100644 --- a/include/xtensor/xtensor_config.hpp +++ b/include/xtensor/xtensor_config.hpp @@ -12,7 +12,7 @@ #define XTENSOR_VERSION_MAJOR 0 #define XTENSOR_VERSION_MINOR 21 -#define XTENSOR_VERSION_PATCH 8 +#define XTENSOR_VERSION_PATCH 9 // DETECT 3.6 <= clang < 3.8 for compiler bug workaround. #ifdef __clang__ From 5b37b84551083ec2d1ef045d6470603c70417b2d Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 6 Nov 2020 00:43:04 +0100 Subject: [PATCH 143/606] Document chunked arrays (#2102) Add xchunked_array documentation --- docs/source/api/chunked_array.rst | 13 +++++ docs/source/api/container_index.rst | 1 + docs/source/index.rst | 1 + docs/source/quickref/basic.rst | 18 +++++-- docs/source/quickref/chunked_arrays.rst | 69 +++++++++++++++++++++++++ include/xtensor/xchunked_array.hpp | 43 ++++++++++++++- 6 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 docs/source/api/chunked_array.rst create mode 100644 docs/source/quickref/chunked_arrays.rst diff --git a/docs/source/api/chunked_array.rst b/docs/source/api/chunked_array.rst new file mode 100644 index 000000000..702b60281 --- /dev/null +++ b/docs/source/api/chunked_array.rst @@ -0,0 +1,13 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +chunked_array +============= + +Defined in ``xtensor/xchunked_array.hpp`` + +.. doxygenfunction:: xt::chunked_array + :project: xtensor diff --git a/docs/source/api/container_index.rst b/docs/source/api/container_index.rst index 081787fb3..29424161c 100644 --- a/docs/source/api/container_index.rst +++ b/docs/source/api/container_index.rst @@ -18,6 +18,7 @@ xexpression API is actually implemented in ``xstrided_container`` and ``xcontain xiterable xarray xarray_adaptor + chunked_array xtensor xtensor_adaptor xfixed diff --git a/docs/source/index.rst b/docs/source/index.rst index 051df2b16..caf071f41 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -87,6 +87,7 @@ for details. view quickref/iterator quickref/manipulation + quickref/chunked_arrays .. toctree:: :caption: API REFERENCE diff --git a/docs/source/quickref/basic.rst b/docs/source/quickref/basic.rst index 5aa105dd0..0a9bea657 100644 --- a/docs/source/quickref/basic.rst +++ b/docs/source/quickref/basic.rst @@ -13,6 +13,7 @@ Tensor types - ``xarray``: tensor that can be reshaped to any number of dimensions. - ``xtensor``: tensor with a number of dimensions set to ``N`` at compile time. - ``xtensor_fixed``: tensor whose shape is fixed at compile time. +- ``xchunked_array``: chunked array using the ``CS`` chunk storage. .. note:: @@ -28,7 +29,7 @@ Tensor with dynamic shape: #include "xarray.hpp" - xt::xarray::shape_type shape = {2, 3}; + xt::xarray::shape_type shape = {2, 3}; xt::xarray a0(shape); xt::xarray a1(shape, 2.5); xt::xarray a2 = {{1., 2., 3.}, {4., 5., 6.}}; @@ -40,12 +41,12 @@ Tensor with static number of dimensions: #include "xtensor.hpp" - xt::xtensor::shape_type shape = {2, 3}; + xt::xtensor::shape_type shape = {2, 3}; xt::xtensor a0(shape); xt::xtensor a1(shape, 2.5); xt::xtensor a2 = {{1., 2., 3.}, {4., 5., 6.}}; auto a3 = xt::xtensor::from_shape(shape); - + Tensor with fixed shape: .. code:: @@ -54,6 +55,16 @@ Tensor with fixed shape: xt::xtensor_fixed> = {{1., 2., 3.}, {4., 5., 6.}}; +In-memory chunked tensor with dynamic shape: + +.. code:: + + #include "xtensor/xchunked_array.hpp" + + std::vector shape = {10, 10, 10}; + std::vector chunk_shape = {2, 3, 4}; + auto a = xt::chunked_array(shape, chunk_shape); + Output ------ @@ -234,4 +245,3 @@ The underlying 1D data buffer can be accessed with the ``data`` method: a.data()[4] = 8.; std::cout << a << std::endl; // Outputs {{1., 2., 3.}, {8., 5., 6.}} - diff --git a/docs/source/quickref/chunked_arrays.rst b/docs/source/quickref/chunked_arrays.rst new file mode 100644 index 000000000..00353197a --- /dev/null +++ b/docs/source/quickref/chunked_arrays.rst @@ -0,0 +1,69 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +Chunked arrays +============== + +Motivation +---------- + +Arrays can be very large and may not fit in memory. In this case, you may not be +able to use an in-memory array such as an ``xarray``. A solution to this problem +is to cut up the large array into many small arrays, called chunks. Not only do +the chunks fit comfortably in memory, but this also allows to process them in +parallel, including in a distributed environment (although this is not supported +yet). + +Formats for the storage of arrays such as `Zarr `_ +specifically target chunked arrays. Such formats are becoming increasingly +popular in the field of big data, since the chunks can be stored in the cloud. + +In-memory chunked arrays +------------------------ + +This may not look very useful at first sight, since each chunk (and thus the +whole array) is hold in memory. It means that it cannot work with very large +arrays, but it may be used to parallelize an algorithm, by processing several +chunks at the same time. + +An in-memory chunked array has the following type: + +.. code:: + + #include "xtensor/xchunked_array.hpp" + + using data_type = double; + // don't use this code: + using inmemory_chunked_array = xt::xchunked_array>>; + +But you should not directly use this type to create a chunked array. Instead, +use the `chunked_array` factory function: + +.. code:: + + #include "xtensor/xchunked_array.hpp" + + std::vector shape = {10, 10, 10}; + std::vector chunk_shape = {2, 3, 4}; + auto a = xt::chunked_array(shape, chunk_shape); + // a is an in-memory chunked array + // each chunk is an xarray, and chunks are hold in an xarray + // thus a is an xarray of xarray elements + a(3, 9, 2) = 1.; // this will address the chunk of index (1, 3, 0) + // and in this chunk, the element of index (1, 0, 2) + +Chunked arrays implement the full semantic of ``xarray``, including lazy +evaluation. + +Stored chunked arrays +--------------------- + +These are arrays whose chunks are stored on a file system, allowing for +persistence of data. In particular, they are used as a building block for the +`xtensor-zarr `_ library. + +For further dedails, please refer to the documentation +of `xtensor-io `_. diff --git a/include/xtensor/xchunked_array.hpp b/include/xtensor/xchunked_array.hpp index 65ce130ca..40c80a07f 100644 --- a/include/xtensor/xchunked_array.hpp +++ b/include/xtensor/xchunked_array.hpp @@ -220,13 +220,52 @@ namespace xt template constexpr bool is_chunked(const xexpression& e); + /** + * Creates an in-memory chunked array. + * This function returns an uninitialized ``xchunked_array>``. + * + * @tparam T The type of the elements (e.g. double) + * @tparam L The layout_type of the array + * @tparam EXT The type of the array extension (default: empty_extension) + * + * @param shape The shape of the array + * @param chunk_shape The shape of a chunk + * @param chunk_memory_layout The layout of each chunk (default: XTENSOR_DEFAULT_LAYOUT) + * + * @return returns a ``xchunked_array>`` with the given shape, chunk shape and memory layout. + */ template xchunked_array>, EXT> chunked_array(S&& shape, S&& chunk_shape, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); + /** + * Creates an in-memory chunked array. + * This function returns a ``xchunked_array>`` initialized from an expression. + * + * @tparam L The layout_type of the array + * @tparam EXT The type of the array extension (default: empty_extension) + * + * @param e The expression to initialize the chunked array from + * @param chunk_shape The shape of a chunk + * @param chunk_memory_layout The layout of each chunk (default: XTENSOR_DEFAULT_LAYOUT) + * + * @return returns a ``xchunked_array>`` from the given expression, with the given chunk shape and memory layout. + */ template xchunked_array>, EXT> chunked_array(const xexpression& e, S&& chunk_shape, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); + /** + * Creates an in-memory chunked array. + * This function returns a ``xchunked_array>`` initialized from an expression. + * + * @tparam L The layout_type of the array + * @tparam EXT The type of the array extension (default: empty_extension) + * + * @param e The expression to initialize the chunked array from + * @param chunk_memory_layout The layout of each chunk (default: XTENSOR_DEFAULT_LAYOUT) + * + * @return returns a ``xchunked_array>`` from the given expression, with the expression's chunk shape and the given memory layout. + */ template xchunked_array>, EXT> chunked_array(const xexpression&e, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); @@ -398,7 +437,7 @@ namespace xt } return this->derived_cast(); } - + template template inline auto xchunked_semantic::operator=(const xexpression& e) -> derived_type& @@ -407,7 +446,7 @@ namespace xt get_assigner(d.chunks()).build_and_assign_temporary(e, d); return d; } - + template template inline auto xchunked_semantic::get_assigner(const CS&) const -> xchunked_assigner From 4b6c840d96929d863e0704bb0abb9239d0820308 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 10 Nov 2020 00:07:10 +0100 Subject: [PATCH 144/606] Removed zarray files --- CMakeLists.txt | 7 - include/xtensor/zarray.hpp | 201 ---------- include/xtensor/zarray_impl.hpp | 390 ------------------ include/xtensor/zassign.hpp | 35 -- include/xtensor/zdispatcher.hpp | 530 ------------------------- include/xtensor/zdispatching_types.hpp | 137 ------- include/xtensor/zfunction.hpp | 159 -------- include/xtensor/zmath.hpp | 195 --------- test/CMakeLists.txt | 1 - test/test_zarray.cpp | 140 ------- 10 files changed, 1795 deletions(-) delete mode 100644 include/xtensor/zarray.hpp delete mode 100644 include/xtensor/zarray_impl.hpp delete mode 100644 include/xtensor/zassign.hpp delete mode 100644 include/xtensor/zdispatcher.hpp delete mode 100644 include/xtensor/zdispatching_types.hpp delete mode 100644 include/xtensor/zfunction.hpp delete mode 100644 include/xtensor/zmath.hpp delete mode 100644 test/test_zarray.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 469262311..91abf63e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,13 +180,6 @@ set(XTENSOR_HEADERS ${XTENSOR_INCLUDE_DIR}/xtensor/xvectorize.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xview.hpp ${XTENSOR_INCLUDE_DIR}/xtensor/xview_utils.hpp - ${XTENSOR_INCLUDE_DIR}/xtensor/zarray.hpp - ${XTENSOR_INCLUDE_DIR}/xtensor/zarray_impl.hpp - ${XTENSOR_INCLUDE_DIR}/xtensor/zassign.hpp - ${XTENSOR_INCLUDE_DIR}/xtensor/zdispatcher.hpp - ${XTENSOR_INCLUDE_DIR}/xtensor/zdispatching_types.hpp - ${XTENSOR_INCLUDE_DIR}/xtensor/zfunction.hpp - ${XTENSOR_INCLUDE_DIR}/xtensor/zmath.hpp ) add_library(xtensor INTERFACE) diff --git a/include/xtensor/zarray.hpp b/include/xtensor/zarray.hpp deleted file mode 100644 index a27ef6c2e..000000000 --- a/include/xtensor/zarray.hpp +++ /dev/null @@ -1,201 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XTENSOR_ZARRAY_HPP -#define XTENSOR_ZARRAY_HPP - -#include - -#include - -#include "xarray.hpp" -#include "zarray_impl.hpp" -#include "zassign.hpp" - -namespace xt -{ - - /********** - * zarray * - **********/ - - class zarray; - - template <> - struct xcontainer_inner_types - { - using temporary_type = zarray; - }; - - class zarray : public xcontainer_semantic - { - public: - - using expression_tag = zarray_expression_tag; - using semantic_base = xcontainer_semantic; - using implementation_ptr = std::unique_ptr; - - zarray() = default; - ~zarray() = default; - - zarray(implementation_ptr&& impl); - zarray& operator=(implementation_ptr&& impl); - - zarray(const zarray& rhs); - zarray& operator=(const zarray& rhs); - - zarray(zarray&& rhs); - zarray& operator=(zarray&& rhs); - - template - zarray(const xexpression& e); - - template - zarray(xexpression& e); - - template - zarray(xexpression&& e); - - template - zarray& operator=(const xexpression&); - - void swap(zarray& rhs); - - zarray_impl& get_implementation(); - const zarray_impl& get_implementation() const; - - template - xarray& get_array(); - - template - const xarray& get_array() const; - - const zchunked_array& as_chunked_array() const; - - private: - - template - void init_implementation(E&& e, xtensor_expression_tag); - - template - void init_implementation(const xexpression& e, zarray_expression_tag); - - implementation_ptr p_impl; - }; - - /************************* - * zarray implementation * - *************************/ - - template - inline void zarray::init_implementation(E&& e, xtensor_expression_tag) - { - p_impl = implementation_ptr(detail::build_zarray(std::forward(e))); - } - - template - inline void zarray::init_implementation(const xexpression& e, zarray_expression_tag) - { - p_impl = nullptr; - semantic_base::assign(e); - } - - inline zarray::zarray(implementation_ptr&& impl) - : p_impl(std::move(impl)) - { - } - - inline zarray& zarray::operator=(implementation_ptr&& impl) - { - p_impl = std::move(impl); - return *this; - } - - inline zarray::zarray(const zarray& rhs) - : p_impl(rhs.p_impl->clone()) - { - } - - inline zarray& zarray::operator=(const zarray& rhs) - { - zarray tmp(rhs); - swap(tmp); - return *this; - } - - inline zarray::zarray(zarray&& rhs) - : p_impl(std::move(rhs.p_impl)) - { - } - - template - inline zarray::zarray(const xexpression& e) - { - init_implementation(e.derived_cast(), extension::get_expression_tag_t>()); - } - - template - inline zarray::zarray(xexpression& e) - { - init_implementation(e.derived_cast(), extension::get_expression_tag_t>()); - } - - template - inline zarray::zarray(xexpression&& e) - { - init_implementation(std::move(e).derived_cast(), extension::get_expression_tag_t>()); - } - - inline zarray& zarray::operator=(zarray&& rhs) - { - swap(rhs); - return *this; - } - - template - inline zarray& zarray::operator=(const xexpression& e) - { - return semantic_base::operator=(e); - } - - inline void zarray::swap(zarray& rhs) - { - std::swap(p_impl, rhs.p_impl); - } - - inline zarray_impl& zarray::get_implementation() - { - return *p_impl; - } - - inline const zarray_impl& zarray::get_implementation() const - { - return *p_impl; - } - - template - inline xarray& zarray::get_array() - { - return dynamic_cast*>(p_impl.get())->get_array(); - } - - template - inline const xarray& zarray::get_array() const - { - return dynamic_cast*>(p_impl.get())->get_array(); - } - - inline const zchunked_array& zarray::as_chunked_array() const - { - return dynamic_cast(*(p_impl.get())); - } -} - -#endif - diff --git a/include/xtensor/zarray_impl.hpp b/include/xtensor/zarray_impl.hpp deleted file mode 100644 index d12ea2b89..000000000 --- a/include/xtensor/zarray_impl.hpp +++ /dev/null @@ -1,390 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XTENSOR_ZARRAY_IMPL_HPP -#define XTENSOR_ZARRAY_IMPL_HPP - -#include "xarray.hpp" -#include "xchunked_array.hpp" - -namespace xt -{ - - /************************* - * zarray_expression_tag * - *************************/ - - struct zarray_expression_tag {}; - - namespace extension - { - template <> - struct expression_tag_and - { - using type = zarray_expression_tag; - }; - - template <> - struct expression_tag_and - : expression_tag_and - { - }; - - template <> - struct expression_tag_and - { - using type = zarray_expression_tag; - }; - } - - /*************** - * zarray_impl * - ***************/ - - class zarray_impl - { - public: - - using self_type = zarray_impl; - - virtual ~zarray_impl() = default; - - zarray_impl(zarray_impl&&) = delete; - zarray_impl& operator=(const zarray_impl&) = delete; - zarray_impl& operator=(zarray_impl&&) = delete; - - virtual self_type* clone() const = 0; - - XTL_IMPLEMENT_INDEXABLE_CLASS() - - protected: - - zarray_impl() = default; - zarray_impl(const zarray_impl&) = default; - }; - - /**************** - * ztyped_array * - ****************/ - - template - class ztyped_array : public zarray_impl - { - public: - - virtual ~ztyped_array() = default; - - virtual xarray& get_array() = 0; - virtual const xarray& get_array() const = 0; - - XTL_IMPLEMENT_INDEXABLE_CLASS() - - protected: - - ztyped_array() = default; - ztyped_array(const ztyped_array&) = default; - }; - - /*********************** - * zexpression_wrapper * - ***********************/ - - template - class zexpression_wrapper : public ztyped_array::value_type> - { - public: - - using self_type = zexpression_wrapper; - using value_type = typename std::decay_t::value_type; - using base_type = ztyped_array; - - template - zexpression_wrapper(E&& e); - - virtual ~zexpression_wrapper() = default; - - xarray& get_array() override; - const xarray& get_array() const override; - - self_type* clone() const override; - - private: - - zexpression_wrapper(const zexpression_wrapper&) = default; - - void compute_cache() const; - - CTE m_expression; - mutable xarray m_cache; - mutable bool m_cache_initialized; - }; - - /****************** - * zarray_wrapper * - ******************/ - - template - class zarray_wrapper : public ztyped_array::value_type> - { - public: - - using self_type = zarray_wrapper; - using value_type = typename std::decay_t::value_type; - using base_type = ztyped_array; - - template - zarray_wrapper(E&& e); - - virtual ~zarray_wrapper() = default; - - xarray& get_array() override; - const xarray& get_array() const override; - - self_type* clone() const override; - - private: - - zarray_wrapper(const zarray_wrapper&) = default; - - CTE m_array; - }; - - /******************** - * zchunked_wrapper * - ********************/ - - class zchunked_array - { - public: - - using shape_type = std::vector; - - virtual ~zchunked_array() = default; - virtual const shape_type& chunk_shape() const = 0; - }; - - template - class zchunked_wrapper : public ztyped_array::value_type>, - public zchunked_array - { - public: - - using self_type = zchunked_wrapper; - using value_type = typename std::decay_t::value_type; - using base_type = ztyped_array; - using shape_type = typename zchunked_array::shape_type; - - template - zchunked_wrapper(E&& e); - - virtual ~zchunked_wrapper() = default; - - xarray& get_array() override; - const xarray& get_array() const override; - - self_type* clone() const override; - - const shape_type& chunk_shape() const override; - - private: - - zchunked_wrapper(const zchunked_wrapper&) = default; - - void compute_cache() const; - - CTE m_chunked_array; - shape_type m_chunk_shape; - mutable xarray m_cache; - mutable bool m_cache_initialized; - - }; - - /*********************** - * zexpression_wrapper * - ***********************/ - - template - template - inline zexpression_wrapper::zexpression_wrapper(E&& e) - : base_type() - , m_expression(std::forward(e)) - , m_cache() - , m_cache_initialized(false) - { - } - - template - inline auto zexpression_wrapper::get_array() -> xarray& - { - compute_cache(); - return m_cache; - } - - template - inline auto zexpression_wrapper::get_array() const -> const xarray& - { - compute_cache(); - return m_cache; - } - - template - inline auto zexpression_wrapper::clone() const -> self_type* - { - return new self_type(*this); - } - - template - inline void zexpression_wrapper::compute_cache() const - { - if (!m_cache_initialized) - { - m_cache = m_expression; - m_cache_initialized = true; - } - } - - /****************** - * zarray_wrapper * - ******************/ - - template - template - inline zarray_wrapper::zarray_wrapper(E&& e) - : base_type() - , m_array(std::forward(e)) - { - } - - template - inline auto zarray_wrapper::get_array() -> xarray& - { - return m_array; - } - - template - inline auto zarray_wrapper::get_array() const -> const xarray& - { - return m_array; - } - - template - inline auto zarray_wrapper::clone() const -> self_type* - { - return new self_type(*this); - } - - /******************** - * zchunked_wrapper * - ********************/ - - template - template - inline zchunked_wrapper::zchunked_wrapper(E&& e) - : base_type() - , m_chunked_array(std::forward(e)) - , m_chunk_shape(m_chunked_array.chunk_shape().size()) - , m_cache() - , m_cache_initialized(false) - { - std::copy(m_chunked_array.chunk_shape().begin(), - m_chunked_array.chunk_shape().end(), - m_chunk_shape.begin()); - } - - template - inline auto zchunked_wrapper::get_array() -> xarray& - { - compute_cache(); - return m_cache; - } - - template - inline auto zchunked_wrapper::get_array() const -> const xarray& - { - compute_cache(); - return m_cache; - } - - template - inline auto zchunked_wrapper::clone() const -> self_type* - { - return new self_type(*this); - } - - template - inline auto zchunked_wrapper::chunk_shape() const -> const shape_type& - { - return m_chunk_shape; - } - - template - inline void zchunked_wrapper::compute_cache() const - { - if (!m_cache_initialized) - { - m_cache = m_chunked_array; - m_cache_initialized = true; - } - } - - /****************** - * zarray builder * - ******************/ - - namespace detail - { - template - struct is_xarray : std::false_type - { - }; - - template - struct is_xarray> : std::true_type - { - }; - - template - struct is_chunked_array : std::false_type - { - }; - - template - struct is_chunked_array> : std::true_type - { - }; - - template - struct zwrapper_builder - { - using closure_type = xtl::closure_type_t; - using wrapper_type = std::conditional_t>::value, - zarray_wrapper, - std::conditional_t>::value, - zchunked_wrapper, - zexpression_wrapper - > - >; - - template - static wrapper_type* run(OE&& e) - { - return new wrapper_type(std::forward(e)); - } - }; - - template - inline auto build_zarray(E&& e) - { - return zwrapper_builder::run(std::forward(e)); - } - } -} - -#endif - diff --git a/include/xtensor/zassign.hpp b/include/xtensor/zassign.hpp deleted file mode 100644 index 36ca622a4..000000000 --- a/include/xtensor/zassign.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XTENSOR_ZASSIGN_HPP -#define XTENSOR_ZASSIGN_HPP - -#include "xassign.hpp" -#include "zarray_impl.hpp" - -namespace xt -{ - template <> - class xexpression_assigner - { - public: - - template - static void assign_xexpression(xexpression& e1, const xexpression& e2) - { - std::unique_ptr res_impl = e2.derived_cast().allocate_result(); - e2.derived_cast().assign_to(*res_impl); - e1.derived_cast() = std::move(res_impl); - } - }; - -} - -#endif - diff --git a/include/xtensor/zdispatcher.hpp b/include/xtensor/zdispatcher.hpp deleted file mode 100644 index 1b7a6752d..000000000 --- a/include/xtensor/zdispatcher.hpp +++ /dev/null @@ -1,530 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XTENSOR_ZDISPATCHER_HPP -#define XTENSOR_ZDISPATCHER_HPP - -#include - -#include "zdispatching_types.hpp" -#include "zmath.hpp" - -namespace xt -{ - namespace mpl = xtl::mpl; - - template - using zrun_dispatcher_impl = xtl::functor_dispatcher - < - type_list, - void, - xtl::static_caster, - xtl::basic_fast_dispatcher - >; - - template - using ztype_dispatcher_impl = xtl::functor_dispatcher - < - type_list, - size_t, - xtl::static_caster, - xtl::basic_fast_dispatcher - >; - - /********************** - * zdouble_dispatcher * - **********************/ - - // Double dispatchers are used for unary operations. - // They dispatch on the single argument and on the - // result. - - template - class zdouble_dispatcher - { - public: - - template - static void insert(); - - template - static void register_dispatching(mpl::vector, U...>); - - static void init(); - static void dispatch(const zarray_impl& z1, zarray_impl& res); - static size_t get_type_index(const zarray_impl& z1); - - private: - - static zdouble_dispatcher& instance(); - - zdouble_dispatcher(); - ~zdouble_dispatcher() = default; - - template - void insert_impl(); - - template - inline void register_dispatching_impl(mpl::vector, U...>); - inline void register_dispatching_impl(mpl::vector<>); - - using zfunctor_type = get_zmapped_functor_t; - using ztype_dispatcher = ztype_dispatcher_impl>; - using zrun_dispatcher = zrun_dispatcher_impl>; - - ztype_dispatcher m_type_dispatcher; - zrun_dispatcher m_run_dispatcher; - }; - - - /********************** - * ztriple_dispatcher * - **********************/ - - // Triple dispatchers are used for binary operations. - // They dispatch on both arguments and on the result. - - template - class ztriple_dispatcher - { - public: - - template - static void insert(); - - template - static void register_dispatching(mpl::vector, U...>); - - static void init(); - static void dispatch(const zarray_impl& z1, const zarray_impl& z2, zarray_impl& res); - static size_t get_type_index(const zarray_impl& z1, const zarray_impl& z2); - - private: - - static ztriple_dispatcher& instance(); - - ztriple_dispatcher(); - ~ztriple_dispatcher() = default; - - template - void insert_impl(); - - template - inline void register_dispatching_impl(mpl::vector, U...>); - inline void register_dispatching_impl(mpl::vector<>); - - using zfunctor_type = get_zmapped_functor_t; - using ztype_dispatcher = ztype_dispatcher_impl>; - using zrun_dispatcher = zrun_dispatcher_impl>; - - ztype_dispatcher m_type_dispatcher; - zrun_dispatcher m_run_dispatcher; - }; - - /*************** - * zdispatcher * - ***************/ - - template - struct zdispatcher; - - template - struct zdispatcher - { - using type = zdouble_dispatcher; - }; - - template - struct zdispatcher - { - using type = ztriple_dispatcher; - }; - - template - using zdispatcher_t = typename zdispatcher::type; - - /************************ - * zarray_impl_register * - ************************/ - - class zarray_impl_register - { - public: - - template - static void insert(); - - static void init(); - static const zarray_impl& get(size_t index); - - private: - - static zarray_impl_register& instance(); - - zarray_impl_register(); - ~zarray_impl_register() = default; - - template - void insert_impl(); - - size_t m_next_index; - std::vector> m_register; - }; - - /**************** - * init_zsystem * - ****************/ - - // Early initialization of all dispatchers - // and zarray_impl_register - // return int so it can be assigned to a - // static variable and be automatically - // called when loading a shared library - // for instance. - - int init_zsystem(); - - /************************************* - * zdouble_dispatcher implementation * - *************************************/ - - namespace detail - { - template - struct unary_dispatching_types - { - using type = zunary_func_types; - }; - - template <> - struct unary_dispatching_types - { - using type = zunary_op_types; - }; - - template <> - struct unary_dispatching_types - { - using type = zunary_op_types; - }; - - template - using unary_dispatching_types_t = typename unary_dispatching_types::type; - } - - template - template - inline void zdouble_dispatcher::insert() - { - instance().template insert_impl(); - } - - template - template - inline void zdouble_dispatcher::register_dispatching(mpl::vector, U...>) - { - instance().register_dispatching_impl(mpl::vector, U...>()); - } - - template - inline void zdouble_dispatcher::init() - { - instance(); - } - - template - inline void zdouble_dispatcher::dispatch(const zarray_impl& z1, zarray_impl& res) - { - instance().m_run_dispatcher.dispatch(z1, res); - } - - template - inline size_t zdouble_dispatcher::get_type_index(const zarray_impl& z1) - { - return instance().m_type_dispatcher.dispatch(z1); - } - - template - inline zdouble_dispatcher& zdouble_dispatcher::instance() - { - static zdouble_dispatcher inst; - return inst; - } - - template - inline zdouble_dispatcher::zdouble_dispatcher() - { - register_dispatching_impl(detail::unary_dispatching_types_t()); - } - - template - template - inline void zdouble_dispatcher::insert_impl() - { - using arg_type = const ztyped_array; - using res_type = ztyped_array; - m_run_dispatcher.template insert(&zfunctor_type::template run); - m_type_dispatcher.template insert(&zfunctor_type::template index); - } - - template - template - inline void zdouble_dispatcher::register_dispatching_impl(mpl::vector, U...>) - { - insert_impl(); - register_dispatching_impl(mpl::vector()); - } - - template - inline void zdouble_dispatcher::register_dispatching_impl(mpl::vector<>) - { - } - - /************************************* - * ztriple_dispatcher implementation * - *************************************/ - - namespace detail - { - using zbinary_func_list = mpl::vector - < - math::atan2_fun, - math::hypot_fun, - math::pow_fun, - math::fdim_fun, - math::fmax_fun, - math::fmin_fun, - math::remainder_fun, - math::fmod_fun - >; - - template - struct binary_dispatching_types - { - using type = std::conditional_t::value, - zbinary_func_types, - zbinary_op_types>; - }; - - template - using binary_dispatching_types_t = typename binary_dispatching_types::type; - } - - template - template - inline void ztriple_dispatcher::insert() - { - instance().template insert_impl(); - } - - template - template - inline void ztriple_dispatcher::register_dispatching(mpl::vector, U...>) - { - instance().register_impl(mpl::vector, U...>()); - } - - template - inline void ztriple_dispatcher::init() - { - instance(); - } - - template - inline void ztriple_dispatcher::dispatch(const zarray_impl& z1, const zarray_impl& z2, zarray_impl& res) - { - instance().m_run_dispatcher.dispatch(z1, z2, res); - } - - template - inline size_t ztriple_dispatcher::get_type_index(const zarray_impl& z1, const zarray_impl& z2) - { - return instance().m_type_dispatcher.dispatch(z1, z2); - } - - template - inline ztriple_dispatcher& ztriple_dispatcher::instance() - { - static ztriple_dispatcher inst; - return inst; - } - - template - inline ztriple_dispatcher::ztriple_dispatcher() - { - register_dispatching_impl(detail::binary_dispatching_types_t()); - } - - template - template - inline void ztriple_dispatcher::insert_impl() - { - using arg_type1 = const ztyped_array; - using arg_type2 = const ztyped_array; - using res_type = ztyped_array; - m_run_dispatcher.template insert(&zfunctor_type::template run); - m_type_dispatcher.template insert(&zfunctor_type::template index); - } - - - template - template - inline void ztriple_dispatcher::register_dispatching_impl(mpl::vector, U...>) - { - insert_impl(); - register_dispatching_impl(mpl::vector()); - } - - template - inline void ztriple_dispatcher::register_dispatching_impl(mpl::vector<>) - { - } - - /*************************************** - * zarray_impl_register implementation * - ***************************************/ - - template - inline void zarray_impl_register::insert() - { - instance().template insert_impl(); - } - - inline void zarray_impl_register::init() - { - instance(); - } - - inline const zarray_impl& zarray_impl_register::get(size_t index) - { - return *(instance().m_register[index]); - } - - inline zarray_impl_register& zarray_impl_register::instance() - { - static zarray_impl_register r; - return r; - } - - inline zarray_impl_register::zarray_impl_register() - : m_next_index(0) - { - insert_impl(); - insert_impl(); - } - - template - inline void zarray_impl_register::insert_impl() - { - size_t& idx = ztyped_array::get_class_static_index(); - if (idx == SIZE_MAX) - { - m_register.resize(++m_next_index); - idx = m_register.size() - 1u; - - } - else if (m_register.size() <= idx) - { - m_register.resize(idx + 1u); - } - m_register[idx] = std::unique_ptr(detail::build_zarray(std::move(xarray()))); - } - - /******************************* - * init_zsystem implementation * - *******************************/ - - namespace detail - { - inline void init_zdispatchers() - { - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - //zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - //zdispatcher_t::init(); - //zdispatcher_t::init(); - //zdispatcher_t::init(); - //zdispatcher_t::init(); - //zdispatcher_t::init(); - //zdispatcher_t::init(); - //zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - //zdispatcher_t::init(); - //zdispatcher_t::init(); - } - } - - namespace math - { - inline void init_zdispatchers() - { - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - /*zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init(); - zdispatcher_t::init();*/ - } - } - - inline int init_zsystem() - { - detail::init_zdispatchers(); - math::init_zdispatchers(); - return 0; - } -} - -#endif diff --git a/include/xtensor/zdispatching_types.hpp b/include/xtensor/zdispatching_types.hpp deleted file mode 100644 index e3afb85fb..000000000 --- a/include/xtensor/zdispatching_types.hpp +++ /dev/null @@ -1,137 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XTENSOR_ZDISPATCHING_TYPES_HPP -#define XTENSOR_ZDISPATCHING_TYPES_HPP - -#include - -namespace xt -{ - namespace mpl = xtl::mpl; - - // TODO: move to XTL - namespace detail - { - template - struct concatenate; - - template - struct concatenate, mpl::vector> - { - using type = mpl::vector; - }; - - template - struct concatenate, mpl::vector, L...> - { - using type = typename concatenate< - typename concatenate< - mpl::vector, - mpl::vector - >::type, - L... - >::type; - }; - - template - using concatenate_t = typename concatenate::type; - } - - /*********** - * z types * - ***********/ - - using z_int_types = mpl::vector; - using z_small_int_types = mpl::vector; - using z_big_int_types = mpl::vector; - using z_float_types = mpl::vector; - - using z_types = detail::concatenate_t; - - /************************* - * unary operation types * - *************************/ - - template - struct build_unary_impl - { - using type = mpl::vector; - }; - - template - using build_unary_impl_t = typename build_unary_impl::type; - - template - using build_unary_identity_t = build_unary_impl_t; - - template - using build_unary_int32_t = build_unary_impl_t; - - template - using build_unary_int64_t = build_unary_impl_t; - - template - using build_unary_double_t = build_unary_impl_t; - - using zunary_func_types = detail::concatenate_t< - mpl::transform_t, - mpl::transform_t - >; - - using zunary_op_types = detail::concatenate_t< - mpl::transform_t, - mpl::transform_t, - mpl::transform_t - >; - - /************************** - * binary operation types * - **************************/ - - template - struct build_binary_impl - { - using type = mpl::vector; - }; - - template - using build_binary_impl_t = typename build_binary_impl::type; - - template - using build_binary_identity_t = build_binary_impl_t; - - template - using build_binary_int32_t = build_binary_impl_t; - - template - using build_binary_int64_t = build_binary_impl_t; - - template - using build_binary_double_t = build_binary_impl_t; - - using zbinary_func_types = detail::concatenate_t< - mpl::transform_t, - mpl::transform_t - >; - - using zbinary_op_types = detail::concatenate_t< - mpl::transform_t, - mpl::transform_t, - mpl::transform_t - >; - -} - -#endif - diff --git a/include/xtensor/zfunction.hpp b/include/xtensor/zfunction.hpp deleted file mode 100644 index 0f6fb55ea..000000000 --- a/include/xtensor/zfunction.hpp +++ /dev/null @@ -1,159 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XTENSOR_ZFUNCTION_HPP -#define XTENSOR_ZFUNCTION_HPP - -#include -#include - -#include "zdispatcher.hpp" - -namespace xt -{ - template - class zfunction : public xexpression> - { - public: - - using expression_tag = zarray_expression_tag; - - using self_type = zfunction; - using tuple_type = std::tuple; - using functor_type = F; - - template , self_type>::value>> - zfunction(Func&& f, CTA&&... e) noexcept; - - std::unique_ptr allocate_result() const; - std::size_t get_result_type_index() const; - zarray_impl& assign_to(zarray_impl& res) const; - - private: - - using dispatcher_type = zdispatcher_t; - - template - std::size_t get_result_type_index_impl(std::index_sequence) const; - - template - zarray_impl& assign_to_impl(std::index_sequence, zarray_impl& res) const; - - tuple_type m_e; - }; - - namespace detail - { - template - struct select_xfunction_expression - { - using type = zfunction; - }; - } - - /**************************** - * zfunction implementation * - ****************************/ - - class zarray; - - namespace detail - { - - template - struct zfunction_argument - { - static std::size_t get_index(const E& e) - { - return e.get_result_type_index(); - } - - static const zarray_impl& get_array_impl(const E& e, zarray_impl& res) - { - return e.assign_to(res); - } - }; - - template <> - struct zfunction_argument - { - template - static std::size_t get_index(const E& e) - { - return e.get_implementation().get_class_index(); - } - - template - static const zarray_impl& get_array_impl(const E& e, zarray_impl&) - { - return e.get_implementation(); - } - }; - - template - inline size_t get_result_type_index(const E& e) - { - return zfunction_argument::get_index(e); - } - - template - inline const zarray_impl& get_array_impl(const E& e, zarray_impl& z) - { - return zfunction_argument::get_array_impl(e, z); - } - } - - template - template - inline zfunction::zfunction(Func&&, CTA&&... e) noexcept - : m_e(std::forward(e)...) - { - } - - template - std::unique_ptr zfunction::allocate_result() const - { - std::size_t idx = get_result_type_index(); - return std::unique_ptr(zarray_impl_register::get(idx).clone()); - } - - template - std::size_t zfunction::get_result_type_index() const - { - return get_result_type_index_impl(std::make_index_sequence()); - } - - template - inline zarray_impl& zfunction::assign_to(zarray_impl& res) const - { - return assign_to_impl(std::make_index_sequence(), res); - } - - template - template - std::size_t zfunction::get_result_type_index_impl(std::index_sequence) const - { - return dispatcher_type::get_type_index( - zarray_impl_register::get( - detail::get_result_type_index(std::get(m_e)) - )... - ); - } - - template - template - inline zarray_impl& zfunction::assign_to_impl(std::index_sequence, zarray_impl& res) const - { - dispatcher_type::dispatch(detail::get_array_impl(std::get(m_e), res)..., res); - return res; - } -} - -#endif - diff --git a/include/xtensor/zmath.hpp b/include/xtensor/zmath.hpp deleted file mode 100644 index d6fba67d0..000000000 --- a/include/xtensor/zmath.hpp +++ /dev/null @@ -1,195 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XTENSOR_ZMATH_HPP -#define XTENSOR_ZMATH_HPP - -#include "xmath.hpp" -#include "zarray_impl.hpp" - -namespace xt -{ - namespace detail - { - // For further improvement: move shape computation - // at the beginning of a zarray assignment so it is computed - // only once - template - inline void zassign_data(xexpression& e1, const xexpression& e2) - { - e1.derived_cast() = e2.derived_cast(); - } - } - - template - struct get_zmapped_functor; - - template - using get_zmapped_functor_t = typename get_zmapped_functor::type; - -#define XTENSOR_ZMAPPED_FUNCTOR(ZFUN, XFUN) \ - template <> \ - struct get_zmapped_functor \ - { using type = ZFUN; } - -#define XTENSOR_UNARY_ZOPERATOR(ZNAME, XOP, XFUN) \ - struct ZNAME \ - { \ - template \ - static void run(const ztyped_array& z, ztyped_array& zres) \ - { \ - detail::zassign_data(zres.get_array(), XOP z.get_array()); \ - } \ - template \ - static size_t index(const ztyped_array&) \ - { \ - using result_type = ztyped_array())>; \ - return result_type::get_class_static_index(); \ - } \ - }; \ - XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - -#define XTENSOR_BINARY_ZOPERATOR(ZNAME, XOP, XFUN) \ - struct ZNAME \ - { \ - template \ - static void run(const ztyped_array& z1, \ - const ztyped_array& z2, \ - ztyped_array& zres) \ - { \ - detail::zassign_data(zres.get_array(), \ - z1.get_array() XOP z2.get_array()); \ - } \ - template \ - static size_t index(const ztyped_array&, const ztyped_array&) \ - { \ - using result_type = \ - ztyped_array() XOP std::declval())>; \ - return result_type::get_class_static_index(); \ - } \ - }; \ - XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - -#define XTENSOR_UNARY_ZFUNCTOR(ZNAME, XEXP, XFUN) \ - struct ZNAME \ - { \ - template \ - static void run(const ztyped_array& z, \ - ztyped_array& zres) \ - { \ - detail::zassign_data(zres.get_array(), XEXP(z.get_array())); \ - } \ - template \ - static size_t index(const ztyped_array&) \ - { \ - using value_type = decltype(std::declval()(std::declval())); \ - return ztyped_array::get_class_static_index(); \ - } \ - }; \ - XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - -#define XTENSOR_BINARY_ZFUNCTOR(ZNAME, XEXP, XFUN) \ - struct ZNAME \ - { \ - template \ - static void run(const ztyped_array& z1, \ - const ztyped_array& z2, \ - ztyped_array& zres) \ - { \ - detail::zassign_data(zres.get_array(), \ - XEXP(z1.get_array(), z2.get_array())); \ - } \ - template \ - static size_t index(const ztyped_array&, const ztyped_array&) \ - { \ - using value_type = decltype( \ - std::declval()(std::declval(), std::declval())); \ - return ztyped_array::get_class_static_index(); \ - } \ - }; \ - XTENSOR_ZMAPPED_FUNCTOR(ZNAME, XFUN) - - XTENSOR_UNARY_ZOPERATOR(zidentity, +, detail::identity); - XTENSOR_UNARY_ZOPERATOR(znegate, -, detail::negate); - XTENSOR_BINARY_ZOPERATOR(zplus, +, detail::plus); - XTENSOR_BINARY_ZOPERATOR(zminus, -, detail::minus); - XTENSOR_BINARY_ZOPERATOR(zmultiuplies, *, detail::multiplies); - XTENSOR_BINARY_ZOPERATOR(zdivides, /, detail::divides); - XTENSOR_BINARY_ZOPERATOR(zmodulus, %, detail::modulus); - XTENSOR_BINARY_ZOPERATOR(zlogical_or, ||, detail::logical_or); - XTENSOR_BINARY_ZOPERATOR(zlogical_and, &&, detail::logical_and); - XTENSOR_UNARY_ZOPERATOR(zlogical_not, !, detail::logical_not); - XTENSOR_BINARY_ZOPERATOR(zbitwise_or, |, detail::bitwise_or); - XTENSOR_BINARY_ZOPERATOR(zbitwise_and, &, detail::bitwise_and); - XTENSOR_BINARY_ZOPERATOR(zbitwise_xor, ^, detail::bitwise_xor); - XTENSOR_UNARY_ZOPERATOR(zbitwise_not, ~, detail::bitwise_not); - XTENSOR_BINARY_ZOPERATOR(zleft_shift, <<, detail::left_shift); - XTENSOR_BINARY_ZOPERATOR(zright_shift, >>, detail::right_shift); - XTENSOR_BINARY_ZOPERATOR(zless, <, detail::less); - XTENSOR_BINARY_ZOPERATOR(zless_equal, <=, detail::less_equal); - XTENSOR_BINARY_ZOPERATOR(zgreater, >, detail::greater); - XTENSOR_BINARY_ZOPERATOR(zgreater_equal, >=, detail::greater_equal); - XTENSOR_BINARY_ZOPERATOR(zequal_to, ==, detail::equal_to); - XTENSOR_BINARY_ZOPERATOR(znot_equal_to, !=, detail::not_equal_to); - - - XTENSOR_UNARY_ZFUNCTOR(zfabs, xt::fabs, math::fabs_fun); - XTENSOR_BINARY_ZFUNCTOR(zfmod, xt::fmod, math::fmod_fun); - XTENSOR_BINARY_ZFUNCTOR(zremainder, xt::remainder, math::remainder_fun); - //XTENSOR_TERNARY_ZFUNCTOR(fma); - XTENSOR_BINARY_ZFUNCTOR(zfmax, xt::fmax, math::fmax_fun); - XTENSOR_BINARY_ZFUNCTOR(zfmin, xt::fmin, math::fmin_fun); - XTENSOR_BINARY_ZFUNCTOR(zfdim, xt::fdim, math::fdim_fun); - XTENSOR_UNARY_ZFUNCTOR(zexp, xt::exp, math::exp_fun); - XTENSOR_UNARY_ZFUNCTOR(zexp2, xt::exp2, math::exp2_fun); - XTENSOR_UNARY_ZFUNCTOR(zexpm1, xt::expm1, math::expm1_fun); - XTENSOR_UNARY_ZFUNCTOR(zlog, xt::log, math::log_fun); - XTENSOR_UNARY_ZFUNCTOR(zlog10, xt::log10, math::log10_fun); - XTENSOR_UNARY_ZFUNCTOR(zlog2, xt::log2, math::log2_fun); - XTENSOR_UNARY_ZFUNCTOR(zlog1p, xt::log1p, math::log1p_fun); - XTENSOR_BINARY_ZFUNCTOR(zpow, xt::pow, math::pow_fun); - XTENSOR_UNARY_ZFUNCTOR(zsqrt, xt::sqrt, math::sqrt_fun); - XTENSOR_UNARY_ZFUNCTOR(zcbrt, xt::cbrt, math::cbrt_fun); - XTENSOR_BINARY_ZFUNCTOR(zhypot, xt::hypot, math::hypot_fun); - XTENSOR_UNARY_ZFUNCTOR(zsin, xt::sin, math::sin_fun); - XTENSOR_UNARY_ZFUNCTOR(zcos, xt::cos, math::cos_fun); - XTENSOR_UNARY_ZFUNCTOR(ztan, xt::tan, math::tan_fun); - XTENSOR_UNARY_ZFUNCTOR(zasin, xt::asin, math::asin_fun); - XTENSOR_UNARY_ZFUNCTOR(zacos, xt::acos, math::acos_fun); - XTENSOR_UNARY_ZFUNCTOR(zatan, xt::atan, math::atan_fun); - XTENSOR_BINARY_ZFUNCTOR(zatan2, xt::atan2, math::atan2_fun); - XTENSOR_UNARY_ZFUNCTOR(zsinh, xt::sinh, math::sinh_fun); - XTENSOR_UNARY_ZFUNCTOR(zcosh, xt::cosh, math::cosh_fun); - XTENSOR_UNARY_ZFUNCTOR(ztanh, xt::tanh, math::tanh_fun); - XTENSOR_UNARY_ZFUNCTOR(zasinh, xt::asinh, math::asinh_fun); - XTENSOR_UNARY_ZFUNCTOR(zacosh, xt::acosh, math::acosh_fun); - XTENSOR_UNARY_ZFUNCTOR(zatanh, xt::atanh, math::atanh_fun); - XTENSOR_UNARY_ZFUNCTOR(zerf, xt::erf, math::erf_fun); - XTENSOR_UNARY_ZFUNCTOR(zerfc, xt::erfc, math::erfc_fun); - XTENSOR_UNARY_ZFUNCTOR(ztgamma, xt::tgamma, math::tgamma_fun); - XTENSOR_UNARY_ZFUNCTOR(zlgamma, xt::lgamma, math::lgamma_fun); - /*XTENSOR_UNARY_ZFUNCTOR(zceil, xt::ceil, math::ceil_fun); - XTENSOR_UNARY_ZFUNCTOR(zfloor, xt::floor, math::floor_fun); - XTENSOR_UNARY_ZFUNCTOR(ztrunc, xt::trunc, math::trunc_fun); - XTENSOR_UNARY_ZFUNCTOR(zround, xt::round, math::round_fun); - XTENSOR_UNARY_ZFUNCTOR(znearbyint, xt::nearbyint, math::nearbyint_fun); - XTENSOR_UNARY_ZFUNCTOR(zrint, xt::rint, math::rint_fun); - XTENSOR_UNARY_ZFUNCTOR(zisfinite, xt::isfinite, math::isfinite_fun); - XTENSOR_UNARY_ZFUNCTOR(zisinf, xt::isinf, math::isinf_fun); - XTENSOR_UNARY_ZFUNCTOR(zisnan, xt::isnan, math::isnan_fun);*/ - -#undef XTENSOR_BINARY_ZFUNCTOR -#undef XTENSOR_UNARY_ZFUNCTOR -#undef XTENSOR_BINARY_ZOPERATOR -#undef XTENSOR_UNARY_ZOPERATOR -#undef XTENSOR_ZMAPPED_FUNCTOR - -} - -#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fbbfe7b5d..ae467b8ab 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -188,7 +188,6 @@ set(COMMON_BASE test_xview.cpp test_xview_semantic.cpp test_xutils.cpp - test_zarray.cpp ) set(XTENSOR_TESTS diff --git a/test/test_zarray.cpp b/test/test_zarray.cpp deleted file mode 100644 index 6597d866b..000000000 --- a/test/test_zarray.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/*************************************************************************** -* Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * -* Copyright (c) QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#include "gtest/gtest.h" -#include "xtensor/zarray.hpp" -#include "xtensor/zfunction.hpp" - -#ifndef XTENSOR_DISABLE_EXCEPTIONS -namespace xt -{ - using namespace xt; - TEST(zarray, value_semantics) - { - xarray a = {{1., 2.}, {3., 4.}}; - xarray ra = {{2., 2.}, {3., 4.}}; - zarray da(a); - da.get_array()(0, 0) = 2.; - - EXPECT_EQ(a, ra); - } - - // TODO : move to dedicated test file - TEST(zarray, dispatching) - { - using dispatcher_type = zdispatcher_t; - dispatcher_type::init(); - - xarray a = {{0.5, 1.5}, {2.5, 3.5}}; - xarray expa = {{std::exp(0.5), std::exp(1.5)}, {std::exp(2.5), std::exp(3.5)}}; - xarray res; - zarray za(a); - zarray zres(res); - - dispatcher_type::dispatch(za.get_implementation(), zres.get_implementation()); - - EXPECT_EQ(expa, res); - } - - // TODO: move to dedicated test file - TEST(zarray, zfunction) - { - using exp_dispatcher_type = zdispatcher_t; - exp_dispatcher_type::init(); - - using add_dispatcher_type = zdispatcher_t; - add_dispatcher_type::init(); - - using nested_zfunction_type = zfunction; - using zfunction_type = zfunction; - - xarray a = {{0.5, 1.5}, {2.5, 3.5}}; - xarray b = {{-0.2, 2.4}, {1.3, 4.7}}; - xarray res; - - zarray za(a); - zarray zb(b); - zarray zres(res); - - zfunction_type f(zplus(), za, nested_zfunction_type(zexp(), zb)); - f.assign_to(zres.get_implementation()); - - auto expected = xarray::from_shape({2, 2}); - std::transform(a.cbegin(), a.cend(), b.cbegin(), expected.begin(), - [](const double& lhs, const double& rhs) { return lhs + std::exp(rhs); }); - - EXPECT_TRUE(all(isclose(res, expected))); - - size_t res_index = f.get_result_type_index(); - EXPECT_EQ(res_index, ztyped_array::get_class_static_index()); - } - - TEST(zarray, operations) - { - using exp_dispatcher_type = zdispatcher_t; - exp_dispatcher_type::init(); - - using add_dispatcher_type = zdispatcher_t; - add_dispatcher_type::init(); - - xarray a = {{0.5, 1.5}, {2.5, 3.5}}; - xarray b = {{-0.2, 2.4}, {1.3, 4.7}}; - xarray res; - - zarray za(a); - zarray zb(b); - zarray zres(res); - - auto f = za + xt::exp(zb); - f.assign_to(zres.get_implementation()); - - auto expected = xarray::from_shape({2, 2}); - std::transform(a.cbegin(), a.cend(), b.cbegin(), expected.begin(), - [](const double& lhs, const double& rhs) { return lhs + std::exp(rhs); }); - - EXPECT_TRUE(all(isclose(res, expected))); - } - - TEST(zarray, assign) - { - using exp_dispatcher_type = zdispatcher_t; - exp_dispatcher_type::init(); - - using add_dispatcher_type = zdispatcher_t; - add_dispatcher_type::init(); - - xarray a = {{0.5, 1.5}, {2.5, 3.5}}; - xarray b = {{-0.2, 2.4}, {1.3, 4.7}}; - - zarray za(a); - zarray zb(b); - - zarray zres = za + xt::exp(zb); - auto expected = xarray::from_shape({2, 2}); - std::transform(a.cbegin(), a.cend(), b.cbegin(), expected.begin(), - [](const double& lhs, const double& rhs) { return lhs + std::exp(rhs); }); - - const auto& res = zres.get_array(); - EXPECT_TRUE(all(isclose(res, expected))); - } - - TEST(zarray, chunked_array) - { - using shape_type = std::vector; - shape_type shape = {10, 10, 10}; - shape_type chunk_shape = {2, 3, 4}; - auto a = chunked_array(shape, chunk_shape); - - zarray za(a); - shape_type res = za.as_chunked_array().chunk_shape(); - EXPECT_EQ(res, chunk_shape); - } -} -#endif - From e0bd285e16dbca51c0a8279ee24b4ff84391b20d Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Tue, 10 Nov 2020 09:22:39 +0100 Subject: [PATCH 145/606] Fixup docs build --- docs/environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/environment.yml b/docs/environment.yml index 21a4fd065..3b9b55633 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -1,8 +1,7 @@ name: xtensor-docs channels: - - QuantStack + - conda-forge dependencies: - breathe - - sphinx=2.4.4 From 73bd889ec831d7760d5965470f4ce71566de0cc5 Mon Sep 17 00:00:00 2001 From: gouarin Date: Tue, 10 Nov 2020 15:44:28 +0100 Subject: [PATCH 146/606] improve xeval --- include/xtensor/xbuilder.hpp | 4 +-- include/xtensor/xeval.hpp | 23 +++------------ include/xtensor/xexpression_traits.hpp | 41 +++++++++++++++++++------- include/xtensor/xpad.hpp | 13 +++----- include/xtensor/xstrided_view.hpp | 2 +- 5 files changed, 41 insertions(+), 42 deletions(-) diff --git a/include/xtensor/xbuilder.hpp b/include/xtensor/xbuilder.hpp index 113f05796..9b29cd1de 100644 --- a/include/xtensor/xbuilder.hpp +++ b/include/xtensor/xbuilder.hpp @@ -147,7 +147,7 @@ namespace xt template inline auto empty_like(const xexpression& e) { - using xtype = temporary_type_t; + using xtype = temporary_type_t; auto res = xtype::from_shape(e.derived_cast().shape()); return res; } @@ -162,7 +162,7 @@ namespace xt template inline auto full_like(const xexpression& e, typename E::value_type fill_value) { - using xtype = temporary_type_t; + using xtype = temporary_type_t; auto res = xtype::from_shape(e.derived_cast().shape()); res.fill(fill_value); return res; diff --git a/include/xtensor/xeval.hpp b/include/xtensor/xeval.hpp index 04fc5fd26..3f0e9c881 100644 --- a/include/xtensor/xeval.hpp +++ b/include/xtensor/xeval.hpp @@ -10,6 +10,7 @@ #ifndef XTENSOR_EVAL_HPP #define XTENSOR_EVAL_HPP +#include "xexpression_traits.hpp" #include "xtensor_forward.hpp" #include "xshape.hpp" @@ -40,28 +41,12 @@ namespace xt } /// @cond DOXYGEN_INCLUDE_SFINAE - template > - inline auto eval(T&& t) - -> std::enable_if_t::value && detail::is_array::value && !detail::is_fixed::value, xtensor::value>> - { - return xtensor::value>(std::forward(t)); - } - - template > - inline auto eval(T&& t) - -> std::enable_if_t::value && !detail::is_array::value && !detail::is_fixed::value, xt::xarray> - { - return xarray(std::forward(t)); - } - - template > + template inline auto eval(T&& t) - -> std::enable_if_t::value && detail::is_fixed::value && !detail::is_array::value, - xt::xtensor_fixed> + -> std::enable_if_t>::value, temporary_type_t> { - return xtensor_fixed(std::forward(t)); + return std::forward(t); } - /// @endcond } #endif diff --git a/include/xtensor/xexpression_traits.hpp b/include/xtensor/xexpression_traits.hpp index 8cea3e367..a396bca0e 100644 --- a/include/xtensor/xexpression_traits.hpp +++ b/include/xtensor/xexpression_traits.hpp @@ -51,7 +51,7 @@ namespace xt template using common_value_type_t = typename common_value_type::type; - + /******************** * common_size_type * ********************/ @@ -89,7 +89,7 @@ namespace xt template using common_difference_type_t = typename common_difference_type::type; - + /****************** * temporary_type * ******************/ @@ -104,7 +104,7 @@ namespace xt }; #if defined(__GNUC__) && (__GNUC__ > 6) -#if __cplusplus == 201703L +#if __cplusplus == 201703L template