From 9ae8b7c8cd7885b296150a0af5ee3075cbc9c45d Mon Sep 17 00:00:00 2001 From: Umar Arshad Date: Mon, 16 Aug 2021 11:17:49 -0400 Subject: [PATCH 1/3] Fix canny by resizing the sigma array to the correct size The otsuThreshold function was creating an empty Array for the sigmas variable and this sometimes failed because the last value was not always written to. This commit adjusts the size of the sigmas array to better match the values that are assigned to it --- src/api/c/canny.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp index 42aa126929..84a8763483 100644 --- a/src/api/c/canny.cpp +++ b/src/api/c/canny.cpp @@ -95,8 +95,8 @@ Array otsuThreshold(const Array& supEdges, const dim4& iDims = supEdges.dims(); - Array sigmas = createEmptyArray(hDims); - + dim4 sigmaDims(NUM_BINS - 1, hDims[1], hDims[2], hDims[3]); + Array sigmas = createEmptyArray(sigmaDims); for (unsigned b = 0; b < (NUM_BINS - 1); ++b) { seqBegin[0].end = static_cast(b); seqRest[0].begin = static_cast(b + 1); From 738cb277c3ad2ea8ab969a284642988d707430f0 Mon Sep 17 00:00:00 2001 From: pradeep Date: Tue, 17 Aug 2021 14:28:24 +0530 Subject: [PATCH 2/3] Fix edgeTracking CPU kernel to handle batch support Prior to this change, edge tracking CPU backend kernel wasn't processing the batch input sets. Thus, the output of corresponding input sets was missing in the array returned by canny API. This is fixed now. Added a batch test for this scenario. --- src/api/c/canny.cpp | 6 ++-- src/backend/cpu/kernel/canny.hpp | 42 +++++++++++++++------------ test/canny.cpp | 50 +++++++++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp index 84a8763483..e87eef712c 100644 --- a/src/api/c/canny.cpp +++ b/src/api/c/canny.cpp @@ -89,6 +89,7 @@ Array otsuThreshold(const Array& supEdges, vector seqBegin(4, af_span); vector seqRest(4, af_span); + vector sliceIndex(4, af_span); seqBegin[0] = af_make_seq(0, static_cast(hDims[0] - 1), 1); seqRest[0] = af_make_seq(0, static_cast(hDims[0] - 1), 1); @@ -129,11 +130,8 @@ Array otsuThreshold(const Array& supEdges, auto op2 = arithOp(qL, qH, tdims); auto sigma = arithOp(sqrd, op2, tdims); - vector sliceIndex(4, af_span); sliceIndex[0] = {double(b), double(b), 1}; - - auto binRes = createSubArray(sigmas, sliceIndex, false); - + auto binRes = createSubArray(sigmas, sliceIndex, false); copyArray(binRes, sigma); } diff --git a/src/backend/cpu/kernel/canny.hpp b/src/backend/cpu/kernel/canny.hpp index 55ff282db7..ebf3474cf8 100644 --- a/src/backend/cpu/kernel/canny.hpp +++ b/src/backend/cpu/kernel/canny.hpp @@ -114,7 +114,7 @@ void nonMaxSuppression(Param output, CParam magnitude, CParam dxParam, } template -void traceEdge(T* out, const T* strong, const T* weak, int t, int width) { +void traceEdge(T* out, const T* strong, const T* weak, int t, int stride1) { if (!out || !strong || !weak) return; const T EDGE = 1; @@ -129,12 +129,12 @@ void traceEdge(T* out, const T* strong, const T* weak, int t, int width) { // get indices of 8 neighbours std::array potentials; - potentials[0] = t - width - 1; // north-west + potentials[0] = t - stride1 - 1; // north-west potentials[1] = potentials[0] + 1; // north potentials[2] = potentials[1] + 1; // north-east potentials[3] = t - 1; // west potentials[4] = t + 1; // east - potentials[5] = t + width - 1; // south-west + potentials[5] = t + stride1 - 1; // south-west potentials[6] = potentials[5] + 1; // south potentials[7] = potentials[6] + 1; // south-east @@ -151,27 +151,33 @@ void traceEdge(T* out, const T* strong, const T* weak, int t, int width) { template void edgeTrackingHysteresis(Param out, CParam strong, CParam weak) { - const af::dim4 dims = strong.dims(); + const af::dim4 dims = strong.dims(); + const dim_t batchCount = dims[2] * dims[3]; + const dim_t jMax = dims[1] - 1; + const dim_t iMax = dims[0] - 1; - dim_t t = dims[0] + - 1; // skip the first coloumn and first element of second coloumn - dim_t jMax = dims[1] - 1; // max Y value to traverse, ignore right coloumn - dim_t iMax = dims[0] - 1; // max X value to traverse, ignore bottom border - - T* optr = out.get(); const T* sptr = strong.get(); const T* wptr = weak.get(); + T* optr = out.get(); - for (dim_t j = 1; j <= jMax; ++j) { - for (dim_t i = 1; i <= iMax; ++i, ++t) { - // if current pixel(sptr) is part of a edge - // and output doesn't have it marked already, - // mark it and trace the pixels from here. - if (sptr[t] > 0 && optr[t] != 1) { - optr[t] = 1; - traceEdge(optr, sptr, wptr, t, dims[0]); + for (dim_t batchId = 0; batchId < batchCount; ++batchId) { + // Skip processing borders + dim_t t = dims[0] + 1; + + for (dim_t j = 1; j <= jMax; ++j) { + for (dim_t i = 1; i <= iMax; ++i, ++t) { + // if current pixel(sptr) is part of a edge + // and output doesn't have it marked already, + // mark it and trace the pixels from here. + if (sptr[t] > 0 && optr[t] != 1) { + optr[t] = 1; + traceEdge(optr, sptr, wptr, t, dims[0]); + } } } + optr += out.strides(2); + sptr += strong.strides(2); + wptr += weak.strides(2); } } } // namespace kernel diff --git a/test/canny.cpp b/test/canny.cpp index 36b50f673f..e00e9b0c30 100644 --- a/test/canny.cpp +++ b/test/canny.cpp @@ -114,7 +114,6 @@ void cannyImageOtsuTest(string pTestFile, bool isColor) { af_array mulArray = 0; af_array outArray = 0; af_array goldArray = 0; - dim_t nElems = 0; inFiles[testId].insert(0, string(TEST_DIR "/CannyEdgeDetector/")); outFiles[testId].insert(0, string(TEST_DIR "/CannyEdgeDetector/")); @@ -129,12 +128,9 @@ void cannyImageOtsuTest(string pTestFile, bool isColor) { ASSERT_SUCCESS( af_load_image_native(&goldArray, outFiles[testId].c_str())); - ASSERT_SUCCESS(af_get_elements(&nElems, goldArray)); - ASSERT_SUCCESS(af_canny(&_outArray, inArray, AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, false)); - unsigned ndims = 0; dim_t dims[4]; @@ -220,3 +216,49 @@ TEST(CannyEdgeDetector, Sobel5x5_Invalid) { ASSERT_SUCCESS(af_release_array(inArray)); } + +template +void cannyImageOtsuBatchTest(string pTestFile, const dim_t targetBatchCount) { + SUPPORTED_TYPE_CHECK(T); + if (noImageIOTests()) return; + + using af::array; + using af::canny; + using af::loadImage; + using af::loadImageNative; + using af::tile; + + vector inDims; + vector inFiles; + vector outSizes; + vector outFiles; + + readImageTests(pTestFile, inDims, inFiles, outSizes, outFiles); + + size_t testCount = inDims.size(); + + for (size_t testId = 0; testId < testCount; ++testId) { + inFiles[testId].insert(0, string(TEST_DIR "/CannyEdgeDetector/")); + outFiles[testId].insert(0, string(TEST_DIR "/CannyEdgeDetector/")); + + af_dtype type = (af_dtype)dtype_traits::af_type; + array readGold = loadImageNative(outFiles[testId].c_str()); + array goldIm = tile(readGold, 1, 1, targetBatchCount); + array readImg = loadImage(inFiles[testId].c_str(), false).as(type); + array inputIm = tile(readImg, 1, 1, targetBatchCount); + + array outIm = + canny(inputIm, AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, false); + outIm *= 255.0; + + ASSERT_IMAGES_NEAR(outIm.as(u8), goldIm, 1.0e-3); + } +} + +TEST(CannyEdgeDetector, BatchofImagesUsingCPPAPI) { + // DO NOT INCREASE BATCH COUNT BEYOND 4 + // This is a limitation on the test assert macro that is saving + // images to disk which can't handle a batch of images. + cannyImageOtsuBatchTest( + string(TEST_DIR "/CannyEdgeDetector/gray.test"), 3); +} From 75bc1d54e92683cb83890d6a779c0362edba3af7 Mon Sep 17 00:00:00 2001 From: pradeep Date: Tue, 17 Aug 2021 19:23:28 +0530 Subject: [PATCH 3/3] Improve canny's otsu helper by precomputing some arrays Co-authored-by: Umar Arshad --- src/api/c/canny.cpp | 92 +++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 50 deletions(-) diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp index e87eef712c..d625360d3b 100644 --- a/src/api/c/canny.cpp +++ b/src/api/c/canny.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ using detail::ireduce; using detail::logicOp; using detail::reduce; using detail::reduce_all; +using detail::scan; using detail::sobelDerivatives; using detail::uchar; using detail::uint; @@ -71,22 +73,14 @@ Array gradientMagnitude(const Array& gx, const Array& gy, } } -Array otsuThreshold(const Array& supEdges, - const unsigned NUM_BINS, const float maxVal) { - Array hist = histogram(supEdges, NUM_BINS, 0, maxVal, false); +Array otsuThreshold(const Array& in, const unsigned NUM_BINS, + const float maxVal) { + Array hist = histogram(in, NUM_BINS, 0, maxVal, false); - const dim4& hDims = hist.dims(); - - // reduce along histogram dimension i.e. 0th dimension - auto totals = reduce(hist, 0); - - // tile histogram total along 0th dimension - auto ttotals = tile(totals, dim4(hDims[0])); - - // pixel frequency probabilities - auto probability = - arithOp(cast(hist), ttotals, hDims); + const dim4& inDims = in.dims(); + const dim4& hDims = hist.dims(); + const dim4 oDims(1, hDims[1], hDims[2], hDims[3]); vector seqBegin(4, af_span); vector seqRest(4, af_span); vector sliceIndex(4, af_span); @@ -94,55 +88,53 @@ Array otsuThreshold(const Array& supEdges, seqBegin[0] = af_make_seq(0, static_cast(hDims[0] - 1), 1); seqRest[0] = af_make_seq(0, static_cast(hDims[0] - 1), 1); - const dim4& iDims = supEdges.dims(); + Array TWOS = createValueArray(oDims, 2.0f); + Array UnitP = createValueArray(oDims, 1.0f); + Array histf = cast(hist); + Array totals = createValueArray(hDims, inDims[0] * inDims[1]); + Array weights = + iota(dim4(NUM_BINS), oDims); // a.k.a histogram shape + + // pixel frequency probabilities + auto freqs = arithOp(histf, totals, hDims); + auto cumFreqs = scan(freqs, 0); + auto oneMCumFreqs = arithOp(UnitP, cumFreqs, hDims); + auto qLqH = arithOp(cumFreqs, oneMCumFreqs, hDims); + auto product = arithOp(weights, freqs, hDims); + auto cumProduct = scan(product, 0); + auto weightedSum = reduce(product, 0); dim4 sigmaDims(NUM_BINS - 1, hDims[1], hDims[2], hDims[3]); Array sigmas = createEmptyArray(sigmaDims); for (unsigned b = 0; b < (NUM_BINS - 1); ++b) { + const dim4 fDims(b + 1, hDims[1], hDims[2], hDims[3]); + const dim4 eDims(NUM_BINS - 1 - b, hDims[1], hDims[2], hDims[3]); + + sliceIndex[0] = {double(b), double(b), 1}; seqBegin[0].end = static_cast(b); seqRest[0].begin = static_cast(b + 1); - auto frontPartition = createSubArray(probability, seqBegin, false); - auto endPartition = createSubArray(probability, seqRest, false); - - auto qL = reduce(frontPartition, 0); - auto qH = reduce(endPartition, 0); - - const dim4 fdims(b + 1, hDims[1], hDims[2], hDims[3]); - const dim4 edims(NUM_BINS - 1 - b, hDims[1], hDims[2], hDims[3]); - - const dim4 tdims(1, hDims[1], hDims[2], hDims[3]); - auto frontWeights = iota(dim4(b + 1), tdims); - auto endWeights = iota(dim4(NUM_BINS - 1 - b), tdims); - auto offsetValues = createValueArray(edims, b + 1); - - endWeights = arithOp(endWeights, offsetValues, edims); - auto __muL = - arithOp(frontPartition, frontWeights, fdims); - auto __muH = arithOp(endPartition, endWeights, edims); - auto _muL = reduce(__muL, 0); - auto _muH = reduce(__muH, 0); - auto muL = arithOp(_muL, qL, tdims); - auto muH = arithOp(_muH, qH, tdims); - auto TWOS = createValueArray(tdims, 2.0f); - auto diff = arithOp(muL, muH, tdims); - auto sqrd = arithOp(diff, TWOS, tdims); - auto op2 = arithOp(qL, qH, tdims); - auto sigma = arithOp(sqrd, op2, tdims); - - sliceIndex[0] = {double(b), double(b), 1}; - auto binRes = createSubArray(sigmas, sliceIndex, false); + auto qL = createSubArray(cumFreqs, sliceIndex, false); + auto qH = arithOp(UnitP, qL, oDims); + auto _muL = createSubArray(cumProduct, sliceIndex, false); + auto _muH = arithOp(weightedSum, _muL, oDims); + auto muL = arithOp(_muL, qL, oDims); + auto muH = arithOp(_muH, qH, oDims); + auto diff = arithOp(muL, muH, oDims); + auto sqrd = arithOp(diff, TWOS, oDims); + auto op2 = createSubArray(qLqH, sliceIndex, false); + auto sigma = arithOp(sqrd, op2, oDims); + + auto binRes = createSubArray(sigmas, sliceIndex, false); copyArray(binRes, sigma); } - dim4 odims = sigmas.dims(); - odims[0] = 1; - Array thresh = createEmptyArray(odims); - Array locs = createEmptyArray(odims); + Array thresh = createEmptyArray(oDims); + Array locs = createEmptyArray(oDims); ireduce(thresh, locs, sigmas, 0); - return cast(tile(locs, dim4(iDims[0], iDims[1], 1, 1))); + return cast(tile(locs, dim4(inDims[0], inDims[1]))); } Array normalize(const Array& supEdges, const float minVal,