From a4661bad46705f80751ee7c59a16071e1aea6dcb Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 27 Apr 2020 22:05:20 +0530 Subject: [PATCH 1/3] Add OpenCL show build log info to debugging docs page --- docs/pages/debugging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/debugging.md b/docs/pages/debugging.md index bf02679796..6712900f74 100644 --- a/docs/pages/debugging.md +++ b/docs/pages/debugging.md @@ -7,7 +7,7 @@ Using Environment Variables * [`AF_PRINT_ERRORS=1`](configuring_environment.htm#af_print_errors) : Makes exception's messages more helpful * [`AF_TRACE=all`](configuring_environment.htm#af_trace): Print ArrayFire message stream to console * [`AF_JIT_KERNEL_TRACE=stdout`](configuring_environment.htm#af_jit_kernel_trace): Writes out source code generated by ArrayFire's JIT to the specified target - + * [`AF_OPENCL_SHOW_BUILD_INFO=1`](configuring_environment.htm#af_opencl_show_build_info): Print OpenCL kernel build log to console Tips in Language Bindings From e4f6ff68d8f5f4d4d5fda4c4d3f280d1de3f4b98 Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 27 Apr 2020 17:28:16 +0530 Subject: [PATCH 2/3] Clean packed dims calculation in cpu fftconvolve Also made the following additional changes - Refactored variable name to be consistent in cpu fftconvolve i.e. camelCase - Fixed header inclusion as per convention used across library - Removed unused header inclusions --- src/backend/cpu/fftconvolve.cpp | 104 +++++++++++++---------------- src/backend/cuda/fftconvolve.cpp | 4 +- src/backend/opencl/fftconvolve.cpp | 3 +- 3 files changed, 49 insertions(+), 62 deletions(-) diff --git a/src/backend/cpu/fftconvolve.cpp b/src/backend/cpu/fftconvolve.cpp index 28eb5584eb..191c806085 100644 --- a/src/backend/cpu/fftconvolve.cpp +++ b/src/backend/cpu/fftconvolve.cpp @@ -7,18 +7,20 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#include + #include #include -#include -#include -#include #include #include -#include #include #include +#include +#include + using af::dim4; +using std::array; using std::ceil; namespace cpu { @@ -29,79 +31,64 @@ Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind) { const dim4& sd = signal.dims(); const dim4& fd = filter.dims(); - dim_t fftScale = 1; - dim4 packed_dims(1, 1, 1, 1); - int fft_dims[baseDim]; - dim4 sig_tmp_dims, sig_tmp_strides; - dim4 filter_tmp_dims, filter_tmp_strides; + dim4 packedDims(1, 1, 1, 1); + array fftDims; // Pack both signal and filter on same memory array, this will ensure // better use of batched FFT capabilities - fft_dims[baseDim - 1] = nextpow2( + fftDims[baseDim - 1] = nextpow2( static_cast(static_cast(ceil(sd[0] / 2.f)) + fd[0] - 1)); - packed_dims[0] = 2 * fft_dims[baseDim - 1]; - fftScale *= fft_dims[baseDim - 1]; + packedDims[0] = 2 * fftDims[baseDim - 1]; + fftScale *= fftDims[baseDim - 1]; for (dim_t k = 1; k < baseDim; k++) { - packed_dims[k] = nextpow2(static_cast(sd[k] + fd[k] - 1)); - fft_dims[baseDim - k - 1] = packed_dims[k]; - fftScale *= fft_dims[baseDim - k - 1]; + packedDims[k] = nextpow2(static_cast(sd[k] + fd[k] - 1)); + fftDims[baseDim - k - 1] = packedDims[k]; + fftScale *= fftDims[baseDim - k - 1]; } dim_t sbatch = 1, fbatch = 1; - for (int k = baseDim; k < 4; k++) { + for (int k = baseDim; k < AF_MAX_DIMS; k++) { sbatch *= sd[k]; fbatch *= fd[k]; } - packed_dims[baseDim] = (sbatch + fbatch); + packedDims[baseDim] = (sbatch + fbatch); - Array packed = createEmptyArray(packed_dims); + Array packed = createEmptyArray(packedDims); - sig_tmp_dims[0] = filter_tmp_dims[0] = packed_dims[0]; - sig_tmp_strides[0] = filter_tmp_strides[0] = 1; - - for (dim_t k = 1; k < 4; k++) { - if (k < baseDim) { - sig_tmp_dims[k] = packed_dims[k]; - filter_tmp_dims[k] = packed_dims[k]; - } else { - sig_tmp_dims[k] = sd[k]; - filter_tmp_dims[k] = fd[k]; - } - - sig_tmp_strides[k] = sig_tmp_strides[k - 1] * sig_tmp_dims[k - 1]; - filter_tmp_strides[k] = - filter_tmp_strides[k - 1] * filter_tmp_dims[k - 1]; - } + dim4 paddedSigDims(packedDims[0], (1 < baseDim ? packedDims[1] : sd[1]), + (2 < baseDim ? packedDims[2] : sd[2]), + (3 < baseDim ? packedDims[3] : sd[3])); + dim4 paddedFilDims(packedDims[0], (1 < baseDim ? packedDims[1] : fd[1]), + (2 < baseDim ? packedDims[2] : fd[2]), + (3 < baseDim ? packedDims[3] : fd[3])); + dim4 paddedSigStrides = calcStrides(paddedSigDims); + dim4 paddedFilStrides = calcStrides(paddedFilDims); // Number of packed complex elements in dimension 0 dim_t sig_half_d0 = divup(sd[0], 2); // Pack signal in a complex matrix where first dimension is half the input // (allows faster FFT computation) and pad array to a power of 2 with 0s - getQueue().enqueue(kernel::packData, packed, sig_tmp_dims, - sig_tmp_strides, signal); + getQueue().enqueue(kernel::packData, packed, paddedSigDims, + paddedSigStrides, signal); // Pad filter array with 0s - const dim_t offset = sig_tmp_strides[3] * sig_tmp_dims[3]; - getQueue().enqueue(kernel::padArray, packed, filter_tmp_dims, - filter_tmp_strides, filter, offset); - - dim4 fftDims(1, 1, 1, 1); - for (int i = 0; i < baseDim; ++i) { fftDims[i] = fft_dims[i]; } + const dim_t offset = paddedSigStrides[3] * paddedSigDims[3]; + getQueue().enqueue(kernel::padArray, packed, paddedFilDims, + paddedFilStrides, filter, offset); // NOLINTNEXTLINE(performance-unnecessary-value-param) - auto upstream_dft = [=](Param packed, const dim4 fftDims) { - int fft_dims[baseDim]; - for (int i = 0; i < baseDim; ++i) { fft_dims[i] = fftDims[i]; } - const dim4 packed_dims = packed.dims(); + auto upstream_dft = [=](Param packed, + const array fftDims) { + const dim4 packedDims = packed.dims(); const dim4 packed_strides = packed.strides(); // Compute forward FFT if (isDouble) { fftw_plan plan = fftw_plan_many_dft( - baseDim, fft_dims, packed_dims[baseDim], + baseDim, fftDims.data(), packedDims[baseDim], reinterpret_cast(packed.get()), nullptr, packed_strides[0], packed_strides[baseDim] / 2, reinterpret_cast(packed.get()), nullptr, @@ -112,7 +99,7 @@ Array fftconvolve(Array const& signal, Array const& filter, fftw_destroy_plan(plan); } else { fftwf_plan plan = fftwf_plan_many_dft( - baseDim, fft_dims, packed_dims[baseDim], + baseDim, fftDims.data(), packedDims[baseDim], reinterpret_cast(packed.get()), nullptr, packed_strides[0], packed_strides[baseDim] / 2, reinterpret_cast(packed.get()), nullptr, @@ -126,20 +113,19 @@ Array fftconvolve(Array const& signal, Array const& filter, getQueue().enqueue(upstream_dft, packed, fftDims); // Multiply filter and signal FFT arrays - getQueue().enqueue(kernel::complexMultiply, packed, sig_tmp_dims, - sig_tmp_strides, filter_tmp_dims, filter_tmp_strides, - kind, offset); + getQueue().enqueue(kernel::complexMultiply, packed, paddedSigDims, + paddedSigStrides, paddedFilDims, paddedFilStrides, kind, + offset); // NOLINTNEXTLINE(performance-unnecessary-value-param) - auto upstream_idft = [=](Param packed, const dim4 fftDims) { - int fft_dims[baseDim]; - for (int i = 0; i < baseDim; ++i) { fft_dims[i] = fftDims[i]; } - const dim4 packed_dims = packed.dims(); + auto upstream_idft = [=](Param packed, + const array fftDims) { + const dim4 packedDims = packed.dims(); const dim4 packed_strides = packed.strides(); // Compute inverse FFT if (isDouble) { fftw_plan plan = fftw_plan_many_dft( - baseDim, fft_dims, packed_dims[baseDim], + baseDim, fftDims.data(), packedDims[baseDim], reinterpret_cast(packed.get()), nullptr, packed_strides[0], packed_strides[baseDim] / 2, reinterpret_cast(packed.get()), nullptr, @@ -150,7 +136,7 @@ Array fftconvolve(Array const& signal, Array const& filter, fftw_destroy_plan(plan); } else { fftwf_plan plan = fftwf_plan_many_dft( - baseDim, fft_dims, packed_dims[baseDim], + baseDim, fftDims.data(), packedDims[baseDim], reinterpret_cast(packed.get()), nullptr, packed_strides[0], packed_strides[baseDim] / 2, reinterpret_cast(packed.get()), nullptr, @@ -183,8 +169,8 @@ Array fftconvolve(Array const& signal, Array const& filter, Array out = createEmptyArray(oDims); getQueue().enqueue(kernel::reorder, out, - packed, filter, sig_half_d0, fftScale, sig_tmp_dims, - sig_tmp_strides, filter_tmp_dims, filter_tmp_strides, + packed, filter, sig_half_d0, fftScale, paddedSigDims, + paddedSigStrides, paddedFilDims, paddedFilStrides, expand, kind); return out; diff --git a/src/backend/cuda/fftconvolve.cpp b/src/backend/cuda/fftconvolve.cpp index 3b6d38ce8a..8340c54757 100644 --- a/src/backend/cuda/fftconvolve.cpp +++ b/src/backend/cuda/fftconvolve.cpp @@ -7,9 +7,9 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ -#include -#include #include + +#include #include #include diff --git a/src/backend/opencl/fftconvolve.cpp b/src/backend/opencl/fftconvolve.cpp index 01707e5099..cda5285064 100644 --- a/src/backend/opencl/fftconvolve.cpp +++ b/src/backend/opencl/fftconvolve.cpp @@ -7,10 +7,11 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#include + #include #include #include -#include #include #include From eeb9638c5e3bcf19c8ab8b7abaf86346dd71bc5a Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 27 Apr 2020 22:01:03 +0530 Subject: [PATCH 3/3] Remove unnecessary template instantiations for fftconvolve --- src/api/c/fftconvolve.cpp | 78 +++++++++++++---------- src/api/c/morph.cpp | 2 +- src/backend/cpu/fftconvolve.cpp | 49 +++++++------- src/backend/cpu/fftconvolve.hpp | 4 +- src/backend/cpu/kernel/fftconvolve.hpp | 31 +++++---- src/backend/cuda/fftconvolve.cpp | 61 +++++++++--------- src/backend/cuda/fftconvolve.hpp | 4 +- src/backend/cuda/kernel/fftconvolve.hpp | 8 ++- src/backend/opencl/fftconvolve.cpp | 71 +++++++++++---------- src/backend/opencl/fftconvolve.hpp | 4 +- src/backend/opencl/kernel/fftconvolve.hpp | 73 ++++++++++++--------- 11 files changed, 206 insertions(+), 179 deletions(-) diff --git a/src/api/c/fftconvolve.cpp b/src/api/c/fftconvolve.cpp index 87dae06c5c..de756f6ff0 100644 --- a/src/api/c/fftconvolve.cpp +++ b/src/api/c/fftconvolve.cpp @@ -6,6 +6,7 @@ * The complete license agreement can be obtained at: * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ + #include #include #include @@ -18,6 +19,10 @@ #include #include +#include +#include +#include + using af::dim4; using detail::arithOp; using detail::Array; @@ -32,14 +37,23 @@ using detail::uchar; using detail::uint; using detail::uintl; using detail::ushort; +using std::conditional; +using std::is_integral; +using std::is_same; using std::max; using std::swap; using std::vector; -template +template static inline af_array fftconvolve_fallback(const af_array signal, const af_array filter, bool expand) { + using convT = + typename conditional::value || is_same::value, + float, double>::type; + using cT = typename conditional::value, cfloat, + cdouble>::type; + const Array S = castArray(signal); const Array F = castArray(filter); const dim4 &sdims = S.dims(); @@ -103,14 +117,13 @@ static inline af_array fftconvolve_fallback(const af_array signal, } } -template +template inline static af_array fftconvolve(const af_array &s, const af_array &f, const bool expand, AF_BATCH_KIND kind) { if (kind == AF_BATCH_DIFF) { - return fftconvolve_fallback(s, f, expand); + return fftconvolve_fallback(s, f, expand); } else { - return getHandle(fftconvolve( + return getHandle(fftconvolve( getArray(s), castArray(f), expand, kind)); } } @@ -149,73 +162,68 @@ af_err fft_convolve(af_array *out, const af_array signal, const af_array filter, const ArrayInfo &sInfo = getInfo(signal); const ArrayInfo &fInfo = getInfo(filter); - af_dtype stype = sInfo.getType(); + af_dtype signalType = sInfo.getType(); + af_dtype filterType = fInfo.getType(); const dim4 &sdims = sInfo.dims(); const dim4 &fdims = fInfo.dims(); AF_BATCH_KIND convBT = identifyBatchKind(sdims, fdims); + ARG_ASSERT(1, (signalType == filterType)); ARG_ASSERT(1, (convBT != AF_BATCH_UNSUPPORTED)); af_array output; - switch (stype) { + switch (signalType) { case f64: - output = - fftconvolve( - signal, filter, expand, convBT); + output = fftconvolve(signal, filter, expand, + convBT); break; case f32: output = - fftconvolve( - signal, filter, expand, convBT); + fftconvolve(signal, filter, expand, convBT); break; case u32: - output = fftconvolve( - signal, filter, expand, convBT); + output = + fftconvolve(signal, filter, expand, convBT); break; case s32: - output = fftconvolve( - signal, filter, expand, convBT); + output = + fftconvolve(signal, filter, expand, convBT); break; case u64: output = - fftconvolve( - signal, filter, expand, convBT); + fftconvolve(signal, filter, expand, convBT); break; case s64: - output = fftconvolve( - signal, filter, expand, convBT); + output = + fftconvolve(signal, filter, expand, convBT); break; case u16: - output = - fftconvolve( - signal, filter, expand, convBT); + output = fftconvolve(signal, filter, expand, + convBT); break; case s16: output = - fftconvolve( - signal, filter, expand, convBT); + fftconvolve(signal, filter, expand, convBT); break; case u8: output = - fftconvolve( - signal, filter, expand, convBT); + fftconvolve(signal, filter, expand, convBT); break; case b8: - output = fftconvolve( - signal, filter, expand, convBT); + output = + fftconvolve(signal, filter, expand, convBT); break; case c32: - output = fftconvolve_fallback( - signal, filter, expand); + output = fftconvolve_fallback(signal, filter, + expand); break; case c64: - output = - fftconvolve_fallback( - signal, filter, expand); + output = fftconvolve_fallback(signal, filter, + expand); break; - default: TYPE_ERROR(1, stype); + default: TYPE_ERROR(1, signalType); } swap(*out, output); } diff --git a/src/api/c/morph.cpp b/src/api/c/morph.cpp index 9a09f910a5..771f0d651a 100644 --- a/src/api/c/morph.cpp +++ b/src/api/c/morph.cpp @@ -80,7 +80,7 @@ static inline af_array morph(const af_array &input, const af_array &mask) { static_cast(seDims[1] % 2 == 0), 0, 0}, {0, 0, 0, 0}, AF_PAD_ZERO); - auto fftConv = fftconvolve; + auto fftConv = fftconvolve; if (isDilation) { Array dft = diff --git a/src/backend/cpu/fftconvolve.cpp b/src/backend/cpu/fftconvolve.cpp index 191c806085..aa22112987 100644 --- a/src/backend/cpu/fftconvolve.cpp +++ b/src/backend/cpu/fftconvolve.cpp @@ -18,6 +18,7 @@ #include #include +#include using af::dim4; using std::array; @@ -25,10 +26,15 @@ using std::ceil; namespace cpu { -template +template Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind) { + using convT = typename std::conditional::value || + std::is_same::value, + float, double>::type; + + constexpr bool IsTypeDouble = std::is_same::value; + const dim4& sd = signal.dims(); const dim4& fd = filter.dims(); dim_t fftScale = 1; @@ -86,7 +92,7 @@ Array fftconvolve(Array const& signal, Array const& filter, const dim4 packedDims = packed.dims(); const dim4 packed_strides = packed.strides(); // Compute forward FFT - if (isDouble) { + if (IsTypeDouble) { fftw_plan plan = fftw_plan_many_dft( baseDim, fftDims.data(), packedDims[baseDim], reinterpret_cast(packed.get()), nullptr, @@ -123,7 +129,7 @@ Array fftconvolve(Array const& signal, Array const& filter, const dim4 packedDims = packed.dims(); const dim4 packed_strides = packed.strides(); // Compute inverse FFT - if (isDouble) { + if (IsTypeDouble) { fftw_plan plan = fftw_plan_many_dft( baseDim, fftDims.data(), packedDims[baseDim], reinterpret_cast(packed.get()), nullptr, @@ -168,34 +174,33 @@ Array fftconvolve(Array const& signal, Array const& filter, Array out = createEmptyArray(oDims); - getQueue().enqueue(kernel::reorder, out, - packed, filter, sig_half_d0, fftScale, paddedSigDims, - paddedSigStrides, paddedFilDims, paddedFilStrides, - expand, kind); + getQueue().enqueue(kernel::reorder, out, packed, filter, + sig_half_d0, fftScale, paddedSigDims, paddedSigStrides, + paddedFilDims, paddedFilStrides, expand, kind); return out; } -#define INSTANTIATE(T, convT, cT, isDouble, roundOut) \ - template Array fftconvolve( \ +#define INSTANTIATE(T) \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); \ - template Array fftconvolve( \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); \ - template Array fftconvolve( \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); -INSTANTIATE(double, double, cdouble, true, false) -INSTANTIATE(float, float, cfloat, false, false) -INSTANTIATE(uint, float, cfloat, false, true) -INSTANTIATE(int, float, cfloat, false, true) -INSTANTIATE(uchar, float, cfloat, false, true) -INSTANTIATE(char, float, cfloat, false, true) -INSTANTIATE(uintl, float, cfloat, false, true) -INSTANTIATE(intl, float, cfloat, false, true) -INSTANTIATE(ushort, float, cfloat, false, true) -INSTANTIATE(short, float, cfloat, false, true) +INSTANTIATE(double) +INSTANTIATE(float) +INSTANTIATE(uint) +INSTANTIATE(int) +INSTANTIATE(uchar) +INSTANTIATE(char) +INSTANTIATE(uintl) +INSTANTIATE(intl) +INSTANTIATE(ushort) +INSTANTIATE(short) } // namespace cpu diff --git a/src/backend/cpu/fftconvolve.hpp b/src/backend/cpu/fftconvolve.hpp index 671e27ac6b..196dec427a 100644 --- a/src/backend/cpu/fftconvolve.hpp +++ b/src/backend/cpu/fftconvolve.hpp @@ -11,9 +11,7 @@ namespace cpu { -template +template Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind); - } diff --git a/src/backend/cpu/kernel/fftconvolve.hpp b/src/backend/cpu/kernel/fftconvolve.hpp index 78205869c7..951ce33641 100644 --- a/src/backend/cpu/kernel/fftconvolve.hpp +++ b/src/backend/cpu/kernel/fftconvolve.hpp @@ -156,11 +156,13 @@ void complexMultiply(Param packed, const af::dim4 sig_dims, } } -template +template void reorderHelper(To* out_ptr, const af::dim4& od, const af::dim4& os, const Ti* in_ptr, const af::dim4& id, const af::dim4& is, const af::dim4& fd, const int half_di0, const int baseDim, const int fftScale, const bool expand) { + constexpr bool RoundResult = std::is_integral::value; + UNUSED(id); for (int d3 = 0; d3 < (int)od[3]; d3++) { for (int d2 = 0; d2 < (int)od[2]; d2++) { @@ -187,7 +189,7 @@ void reorderHelper(To* out_ptr, const af::dim4& od, const af::dim4& os, if (id0 < half_di0) { // Copy top elements int iidx = id3 + id2 + id1 + id0 * 2; - if (roundOut) + if (RoundResult) out_ptr[oidx] = (To)roundf((float)(in_ptr[iidx] / fftScale)); else @@ -196,7 +198,7 @@ void reorderHelper(To* out_ptr, const af::dim4& od, const af::dim4& os, // Add signal and filter elements to central part int iidx1 = id3 + id2 + id1 + id0 * 2; int iidx2 = id3 + id2 + id1 + (id0 - half_di0) * 2 + 1; - if (roundOut) + if (RoundResult) out_ptr[oidx] = (To)roundf( (float)((in_ptr[iidx1] + in_ptr[iidx2]) / fftScale)); @@ -207,7 +209,7 @@ void reorderHelper(To* out_ptr, const af::dim4& od, const af::dim4& os, // Copy bottom elements const int iidx = id3 + id2 + id1 + (id0 - half_di0) * 2 + 1; - if (roundOut) + if (RoundResult) out_ptr[oidx] = (To)roundf((float)(in_ptr[iidx] / fftScale)); else @@ -219,12 +221,16 @@ void reorderHelper(To* out_ptr, const af::dim4& od, const af::dim4& os, } } -template +template void reorder(Param out, Param packed, CParam filter, const dim_t sig_half_d0, const dim_t fftScale, const dim4 sig_tmp_dims, const dim4 sig_tmp_strides, const dim4 filter_tmp_dims, const dim4 filter_tmp_strides, bool expand, AF_BATCH_KIND kind) { + // TODO(pradeep) check if we can avoid convT template parameter also + // using convT = typename std::conditional::value, + // float, double>::type; + T* out_ptr = out.get(); const af::dim4 out_dims = out.dims(); const af::dim4 out_strides = out.strides(); @@ -237,15 +243,14 @@ void reorder(Param out, Param packed, CParam filter, // Reorder the output if (kind == AF_BATCH_RHS) { - reorderHelper( - out_ptr, out_dims, out_strides, filter_tmp_ptr, filter_tmp_dims, - filter_tmp_strides, filter_dims, sig_half_d0, baseDim, fftScale, - expand); + reorderHelper(out_ptr, out_dims, out_strides, filter_tmp_ptr, + filter_tmp_dims, filter_tmp_strides, + filter_dims, sig_half_d0, baseDim, fftScale, + expand); } else { - reorderHelper( - out_ptr, out_dims, out_strides, sig_tmp_ptr, sig_tmp_dims, - sig_tmp_strides, filter_dims, sig_half_d0, baseDim, fftScale, - expand); + reorderHelper(out_ptr, out_dims, out_strides, sig_tmp_ptr, + sig_tmp_dims, sig_tmp_strides, filter_dims, + sig_half_d0, baseDim, fftScale, expand); } } diff --git a/src/backend/cuda/fftconvolve.cpp b/src/backend/cuda/fftconvolve.cpp index 8340c54757..8316ab26c3 100644 --- a/src/backend/cuda/fftconvolve.cpp +++ b/src/backend/cuda/fftconvolve.cpp @@ -10,12 +10,16 @@ #include #include +#include #include #include -#include +#include using af::dim4; +using std::conditional; +using std::is_integral; +using std::is_same; namespace cuda { @@ -43,10 +47,15 @@ dim4 calcPackedSize(Array const& i1, Array const& i2, return dim4(pd[0], pd[1], pd[2], pd[3]); } -template +template Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind) { + using convT = + typename conditional::value || is_same::value, + float, double>::type; + using cT = typename conditional::value, cfloat, + cdouble>::type; + const dim4& sDims = signal.dims(); const dim4& fDims = filter.dims(); @@ -82,47 +91,37 @@ Array fftconvolve(Array const& signal, Array const& filter, if (kind == AF_BATCH_RHS) { fft_inplace(filter_packed); - if (expand) { - kernel::reorderOutputHelper( - out, filter_packed, signal, filter); - } else { - kernel::reorderOutputHelper( - out, filter_packed, signal, filter); - } + kernel::reorderOutputHelper(out, filter_packed, signal, filter, + expand, baseDim); } else { fft_inplace(signal_packed); - if (expand) { - kernel::reorderOutputHelper( - out, signal_packed, signal, filter); - } else { - kernel::reorderOutputHelper( - out, signal_packed, signal, filter); - } + kernel::reorderOutputHelper(out, signal_packed, signal, filter, + expand, baseDim); } return out; } -#define INSTANTIATE(T, convT, cT, isDouble, roundOut) \ - template Array fftconvolve( \ +#define INSTANTIATE(T) \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); \ - template Array fftconvolve( \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); \ - template Array fftconvolve( \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); -INSTANTIATE(double, double, cdouble, true, false) -INSTANTIATE(float, float, cfloat, false, false) -INSTANTIATE(uint, float, cfloat, false, true) -INSTANTIATE(int, float, cfloat, false, true) -INSTANTIATE(uchar, float, cfloat, false, true) -INSTANTIATE(char, float, cfloat, false, true) -INSTANTIATE(ushort, float, cfloat, false, true) -INSTANTIATE(short, float, cfloat, false, true) -INSTANTIATE(uintl, float, cfloat, false, true) -INSTANTIATE(intl, float, cfloat, false, true) +INSTANTIATE(double) +INSTANTIATE(float) +INSTANTIATE(uint) +INSTANTIATE(int) +INSTANTIATE(uchar) +INSTANTIATE(char) +INSTANTIATE(uintl) +INSTANTIATE(intl) +INSTANTIATE(ushort) +INSTANTIATE(short) } // namespace cuda diff --git a/src/backend/cuda/fftconvolve.hpp b/src/backend/cuda/fftconvolve.hpp index 86748ea16a..04df117831 100644 --- a/src/backend/cuda/fftconvolve.hpp +++ b/src/backend/cuda/fftconvolve.hpp @@ -11,9 +11,7 @@ namespace cuda { -template +template Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind); - } diff --git a/src/backend/cuda/kernel/fftconvolve.hpp b/src/backend/cuda/kernel/fftconvolve.hpp index 52fe80cb4d..eb147a5f64 100644 --- a/src/backend/cuda/kernel/fftconvolve.hpp +++ b/src/backend/cuda/kernel/fftconvolve.hpp @@ -101,13 +101,15 @@ void complexMultiplyHelper(Param sig_packed, Param filter_packed, POST_LAUNCH_CHECK(); } -template +template void reorderOutputHelper(Param out, Param packed, CParam sig, - CParam filter) { + CParam filter, bool expand, int baseDim) { + constexpr bool RoundResult = std::is_integral::value; + auto reorderOut = getKernel("cuda::reorderOutput", fftConvSource(), {TemplateTypename(), TemplateTypename(), - TemplateArg(expand), TemplateArg(roundOut)}); + TemplateArg(expand), TemplateArg(RoundResult)}); dim_t *sd = sig.dims; int fftScale = 1; diff --git a/src/backend/opencl/fftconvolve.cpp b/src/backend/opencl/fftconvolve.cpp index cda5285064..2d090a0b0e 100644 --- a/src/backend/opencl/fftconvolve.cpp +++ b/src/backend/opencl/fftconvolve.cpp @@ -10,12 +10,20 @@ #include #include -#include #include #include #include +#include +#include +#include + using af::dim4; +using std::ceil; +using std::conditional; +using std::is_integral; +using std::is_same; +using std::vector; namespace opencl { @@ -30,7 +38,7 @@ static dim4 calcPackedSize(Array const& i1, Array const& i2, // Pack both signal and filter on same memory array, this will ensure // better use of batched cuFFT capabilities pd[0] = nextpow2(static_cast( - static_cast(std::ceil(i1d[0] / 2.f)) + i2d[0] - 1)); + static_cast(ceil(i1d[0] / 2.f)) + i2d[0] - 1)); for (dim_t k = 1; k < baseDim; k++) { pd[k] = nextpow2(static_cast(i1d[k] + i2d[k] - 1)); @@ -47,10 +55,15 @@ static dim4 calcPackedSize(Array const& i1, Array const& i2, return dim4(pd[0], pd[1], pd[2], pd[3]); } -template +template Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind) { + using convT = + typename conditional::value || is_same::value, + float, double>::type; + using cT = typename conditional::value, cfloat, + cdouble>::type; + const dim4& sDims = signal.dims(); const dim4& fDims = filter.dims(); @@ -73,17 +86,13 @@ Array fftconvolve(Array const& signal, Array const& filter, const dim4 pDims = calcPackedSize(signal, filter, baseDim); Array packed = createEmptyArray(pDims); - kernel::packDataHelper(packed, signal, filter, - baseDim, kind); - + kernel::packDataHelper(packed, signal, filter, baseDim, kind); fft_inplace(packed); - - kernel::complexMultiplyHelper( - packed, signal, filter, baseDim, kind); + kernel::complexMultiplyHelper(packed, signal, filter, baseDim, kind); // Compute inverse FFT only on complex-multiplied data if (kind == AF_BATCH_RHS) { - std::vector seqs; + vector seqs; for (dim_t k = 0; k < 4; k++) { if (k < baseDim) { seqs.push_back({0., static_cast(pDims[k] - 1), 1.}); @@ -97,7 +106,7 @@ Array fftconvolve(Array const& signal, Array const& filter, Array subPacked = createSubArray(packed, seqs); fft_inplace(subPacked); } else { - std::vector seqs; + vector seqs; for (dim_t k = 0; k < 4; k++) { if (k < baseDim) { seqs.push_back({0., static_cast(pDims[k]) - 1, 1.}); @@ -114,37 +123,31 @@ Array fftconvolve(Array const& signal, Array const& filter, Array out = createEmptyArray(oDims); - if (expand) { - kernel::reorderOutputHelper( - out, packed, signal, filter, baseDim, kind); - } else { - kernel::reorderOutputHelper( - out, packed, signal, filter, baseDim, kind); - } - + kernel::reorderOutputHelper(out, packed, signal, filter, baseDim, + kind, expand); return out; } -#define INSTANTIATE(T, convT, cT, isDouble, roundOut) \ - template Array fftconvolve( \ +#define INSTANTIATE(T) \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); \ - template Array fftconvolve( \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); \ - template Array fftconvolve( \ + template Array fftconvolve( \ Array const& signal, Array const& filter, const bool expand, \ AF_BATCH_KIND kind); -INSTANTIATE(double, double, cdouble, true, false) -INSTANTIATE(float, float, cfloat, false, false) -INSTANTIATE(uint, float, cfloat, false, true) -INSTANTIATE(int, float, cfloat, false, true) -INSTANTIATE(uchar, float, cfloat, false, true) -INSTANTIATE(char, float, cfloat, false, true) -INSTANTIATE(ushort, float, cfloat, false, true) -INSTANTIATE(short, float, cfloat, false, true) -INSTANTIATE(uintl, float, cfloat, false, true) -INSTANTIATE(intl, float, cfloat, false, true) +INSTANTIATE(double) +INSTANTIATE(float) +INSTANTIATE(uint) +INSTANTIATE(int) +INSTANTIATE(uchar) +INSTANTIATE(char) +INSTANTIATE(uintl) +INSTANTIATE(intl) +INSTANTIATE(ushort) +INSTANTIATE(short) } // namespace opencl diff --git a/src/backend/opencl/fftconvolve.hpp b/src/backend/opencl/fftconvolve.hpp index ca3d9defa0..0267ad6e85 100644 --- a/src/backend/opencl/fftconvolve.hpp +++ b/src/backend/opencl/fftconvolve.hpp @@ -11,9 +11,7 @@ namespace opencl { -template +template Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind); - } diff --git a/src/backend/opencl/kernel/fftconvolve.hpp b/src/backend/opencl/kernel/fftconvolve.hpp index 7494fc92dd..535ee7c4cc 100644 --- a/src/backend/opencl/kernel/fftconvolve.hpp +++ b/src/backend/opencl/kernel/fftconvolve.hpp @@ -7,6 +7,8 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#pragma once + #include #include #include @@ -17,8 +19,11 @@ #include #include #include +#include #include +#include + using cl::Buffer; using cl::EnqueueArgs; using cl::Kernel; @@ -67,13 +72,15 @@ void calcParamSizes(Param& sig_tmp, Param& filter_tmp, Param& packed, } } -template +template void packDataHelper(Param packed, Param sig, Param filter, const int baseDim, AF_BATCH_KIND kind) { + constexpr bool IsTypeDouble = std::is_same::value; + std::string refName = std::string("pack_data_") + std::string(dtype_traits::getName()) + std::string(dtype_traits::getName()) + - std::to_string(isDouble); + std::to_string(IsTypeDouble); int device = getActiveDeviceId(); kc_entry_t pdkEntry = kernelCache(device, refName); @@ -82,13 +89,13 @@ void packDataHelper(Param packed, Param sig, Param filter, const int baseDim, std::ostringstream options; options << " -D T=" << dtype_traits::getName(); + options << getTypeBuildDefinition(); - if (static_cast(dtype_traits::af_type) == c32) { + auto ctDType = static_cast(dtype_traits::af_type); + if (ctDType == c32) { options << " -D CONVT=float"; - } else if (static_cast(dtype_traits::af_type) == c64 && - isDouble) { - options << " -D CONVT=double" - << " -D USE_DOUBLE"; + } else if (ctDType == c64 && IsTypeDouble) { + options << " -D CONVT=double"; } const char* ker_strs[] = {fftconvolve_pack_cl}; @@ -132,7 +139,7 @@ void packDataHelper(Param packed, Param sig, Param filter, const int baseDim, refName = std::string("pack_array_") + std::string(dtype_traits::getName()) + std::string(dtype_traits::getName()) + - std::to_string(isDouble); + std::to_string(IsTypeDouble); kc_entry_t pakEntry = kernelCache(device, refName); @@ -140,13 +147,13 @@ void packDataHelper(Param packed, Param sig, Param filter, const int baseDim, std::ostringstream options; options << " -D T=" << dtype_traits::getName(); + options << getTypeBuildDefinition(); - if (static_cast(dtype_traits::af_type) == c32) { + auto ctDType = static_cast(dtype_traits::af_type); + if (ctDType == c32) { options << " -D CONVT=float"; - } else if (static_cast(dtype_traits::af_type) == c64 && - isDouble) { - options << " -D CONVT=double" - << " -D USE_DOUBLE"; + } else if (ctDType == c64 && IsTypeDouble) { + options << " -D CONVT=double"; } const char* ker_strs[] = {fftconvolve_pack_cl}; @@ -171,13 +178,15 @@ void packDataHelper(Param packed, Param sig, Param filter, const int baseDim, CL_DEBUG_FINISH(getQueue()); } -template +template void complexMultiplyHelper(Param packed, Param sig, Param filter, const int baseDim, AF_BATCH_KIND kind) { + constexpr bool IsTypeDouble = std::is_same::value; + std::string refName = std::string("complex_multiply_") + std::string(dtype_traits::getName()) + std::string(dtype_traits::getName()) + - std::to_string(isDouble); + std::to_string(IsTypeDouble); int device = getActiveDeviceId(); kc_entry_t entry = kernelCache(device, refName); @@ -190,13 +199,13 @@ void complexMultiplyHelper(Param packed, Param sig, Param filter, << " -D AF_BATCH_LHS=" << (int)AF_BATCH_LHS << " -D AF_BATCH_RHS=" << (int)AF_BATCH_RHS << " -D AF_BATCH_SAME=" << (int)AF_BATCH_SAME; + options << getTypeBuildDefinition(); - if (static_cast(dtype_traits::af_type) == c32) { + auto ctDType = static_cast(dtype_traits::af_type); + if (ctDType == c32) { options << " -D CONVT=float"; - } else if (static_cast(dtype_traits::af_type) == c64 && - isDouble) { - options << " -D CONVT=double" - << " -D USE_DOUBLE"; + } else if (ctDType == c64 && IsTypeDouble) { + options << " -D CONVT=double"; } const char* ker_strs[] = {fftconvolve_multiply_cl}; @@ -234,15 +243,17 @@ void complexMultiplyHelper(Param packed, Param sig, Param filter, CL_DEBUG_FINISH(getQueue()); } -template +template void reorderOutputHelper(Param out, Param packed, Param sig, Param filter, - const int baseDim, AF_BATCH_KIND kind) { + const int baseDim, AF_BATCH_KIND kind, bool expand) { + constexpr bool IsTypeDouble = std::is_same::value; + constexpr bool RoundResult = std::is_integral::value; + std::string refName = std::string("reorder_output_") + std::string(dtype_traits::getName()) + std::string(dtype_traits::getName()) + - std::to_string(isDouble) + std::to_string(roundOut) + - std::to_string(expand); + std::to_string(IsTypeDouble) + + std::to_string(RoundResult) + std::to_string(expand); int device = getActiveDeviceId(); kc_entry_t entry = kernelCache(device, refName); @@ -251,15 +262,15 @@ void reorderOutputHelper(Param out, Param packed, Param sig, Param filter, std::ostringstream options; options << " -D T=" << dtype_traits::getName() - << " -D ROUND_OUT=" << (int)roundOut + << " -D ROUND_OUT=" << (int)RoundResult << " -D EXPAND=" << (int)expand; + options << getTypeBuildDefinition(); - if (static_cast(dtype_traits::af_type) == c32) { + auto ctDType = static_cast(dtype_traits::af_type); + if (ctDType == c32) { options << " -D CONVT=float"; - } else if (static_cast(dtype_traits::af_type) == c64 && - isDouble) { - options << " -D CONVT=double" - << " -D USE_DOUBLE"; + } else if (ctDType == c64 && IsTypeDouble) { + options << " -D CONVT=double"; } const char* ker_strs[] = {fftconvolve_reorder_cl};