Merge pull request #21119 from anna-khakimova:ak/simd_addc

Anna Khakimova · web-flow · commit d58b5ef74b91 · 2021-11-29T11:20:53.000Z
* GAPI Fluid: SIMD for AddC kernel

* Final version

* Applied comments.
diff --git a/modules/gapi/include/opencv2/gapi/core.hpp b/modules/gapi/include/opencv2/gapi/core.hpp
@@ -57,6 +57,7 @@ namespace core {
 
     G_TYPED_KERNEL(GAddC, <GMat(GMat, GScalar, int)>, "org.opencv.core.math.addC") {
         static GMatDesc outMeta(GMatDesc a, GScalarDesc, int ddepth) {
+            GAPI_Assert(a.chan <= 4);
             return a.withDepth(ddepth);
         }
     };
diff --git a/modules/gapi/perf/common/gapi_core_perf_tests.hpp b/modules/gapi/perf/common/gapi_core_perf_tests.hpp
@@ -28,7 +28,7 @@ namespace opencv_test
 //------------------------------------------------------------------------------
 
     class AddPerfTest : public TestPerfParams<tuple<cv::Size, MatType, int, cv::GCompileArgs>> {};
-    class AddCPerfTest : public TestPerfParams<tuple<cv::Size, MatType, int, cv::GCompileArgs>> {};
+    class AddCPerfTest : public TestPerfParams<tuple<compare_f, cv::Size, MatType, int, cv::GCompileArgs>> {};
     class SubPerfTest : public TestPerfParams<tuple<cv::Size, MatType, int, cv::GCompileArgs>> {};
     class SubCPerfTest : public TestPerfParams<tuple<cv::Size, MatType, int, cv::GCompileArgs>> {};
     class SubRCPerfTest : public TestPerfParams<tuple<cv::Size, MatType, int, cv::GCompileArgs>> {};
diff --git a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp
@@ -61,10 +61,13 @@ PERF_TEST_P_(AddPerfTest, TestPerformance)
 
 PERF_TEST_P_(AddCPerfTest, TestPerformance)
 {
-    Size sz = get<0>(GetParam());
-    MatType type = get<1>(GetParam());
-    int dtype = get<2>(GetParam());
-    cv::GCompileArgs compile_args = get<3>(GetParam());
+    compare_f cmpF;
+    cv::Size sz;
+    MatType type = -1;
+    int dtype = -1;
+    cv::GCompileArgs compile_args;
+
+    std::tie(cmpF, sz, type, dtype, compile_args) = GetParam();
 
     initMatsRandU(type, sz, dtype, false);
 
@@ -88,8 +91,9 @@ PERF_TEST_P_(AddCPerfTest, TestPerformance)
     }
 
     // Comparison ////////////////////////////////////////////////////////////
-    // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv));
-    EXPECT_EQ(out_mat_gapi.size(), sz);
+    {
+        EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv));
+    }
 
     SANITY_CHECK_NOTHING();
 }
diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp
@@ -22,7 +22,8 @@ INSTANTIATE_TEST_CASE_P(AddPerfTestCPU, AddPerfTest,
         Values(cv::compile_args(CORE_CPU))));
 
 INSTANTIATE_TEST_CASE_P(AddCPerfTestCPU, AddCPerfTest,
-    Combine(Values(szSmall128, szVGA, sz720p, sz1080p),
+    Combine(Values(AbsExact().to_compare_f()),
+        Values(szSmall128, szVGA, sz720p, sz1080p),
         Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1),
         Values(-1, CV_8U, CV_16U, CV_32F),
         Values(cv::compile_args(CORE_CPU))));
diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp
@@ -18,11 +18,12 @@ INSTANTIATE_TEST_CASE_P(AddPerfTestFluid, AddPerfTest,
             Values(-1, CV_8U, CV_32F),
             Values(cv::compile_args(CORE_FLUID))));
 
-// INSTANTIATE_TEST_CASE_P(AddCPerfTestFluid, AddCPerfTest,
-//     Combine(Values(szSmall128, szVGA, sz720p, sz1080p),
-//         Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1),
-//         Values(-1, CV_8U, CV_16U, CV_32F),
-//         Values(cv::compile_args(CORE_FLUID))));
+ INSTANTIATE_TEST_CASE_P(AddCPerfTestFluid, AddCPerfTest,
+     Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()),
+         Values(szSmall128, szVGA, sz720p, sz1080p),
+         Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1),
+         Values(-1, CV_8U, CV_16U, CV_16S, CV_32F),
+         Values(cv::compile_args(CORE_FLUID))));
 
 INSTANTIATE_TEST_CASE_P(SubPerfTestFluid, SubPerfTest,
     Combine(Values(szSmall128, szVGA, sz720p, sz1080p),
diff --git a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp
@@ -20,7 +20,8 @@ INSTANTIATE_TEST_CASE_P(AddPerfTestGPU, AddPerfTest,
                                 Values(cv::compile_args(CORE_GPU))));
 
 INSTANTIATE_TEST_CASE_P(AddCPerfTestGPU, AddCPerfTest,
-                        Combine(Values( szSmall128, szVGA, sz720p, sz1080p ),
+                        Combine(Values(AbsExact().to_compare_f()),
+                                Values( szSmall128, szVGA, sz720p, sz1080p ),
                                 Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ),
                                 Values( -1, CV_8U, CV_16U, CV_32F ),
                                 Values(cv::compile_args(CORE_GPU))));
diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp
@@ -645,8 +645,8 @@ CV_ALWAYS_INLINE int sub_simd(const SRC in1[], const SRC in2[], DST out[], int l
 #endif // CV_SIMD
 
 template<typename DST, typename SRC1, typename SRC2>
-static void run_arithm(Buffer &dst, const View &src1, const View &src2, Arithm arithm,
-                       double scale=1)
+static CV_ALWAYS_INLINE void run_arithm(Buffer &dst, const View &src1, const View &src2,
+                                        Arithm arithm, double scale=1)
 {
     static_assert(std::is_same<SRC1, SRC2>::value, "wrong types");
 
@@ -844,19 +844,15 @@ GAPI_FLUID_KERNEL(GFluidAbsDiff, cv::gapi::core::GAbsDiff, false)
 //
 //--------------------------------------
 
-static inline v_uint16x8  v_add_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return x + y; }
 static inline v_uint16x8  v_sub_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return x - y; }
 static inline v_uint16x8 v_subr_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return y - x; }
 
-static inline v_float32x4  v_add_32f(const v_float32x4 &x, const v_float32x4 &y) { return x + y; }
 static inline v_float32x4  v_sub_32f(const v_float32x4 &x, const v_float32x4 &y) { return x - y; }
 static inline v_float32x4 v_subr_32f(const v_float32x4 &x, const v_float32x4 &y) { return y - x; }
 
-static inline int  s_add_8u(uchar x, uchar y) { return x + y; }
 static inline int  s_sub_8u(uchar x, uchar y) { return x - y; }
 static inline int s_subr_8u(uchar x, uchar y) { return y - x; }
 
-static inline float  s_add_32f(float x, float y) { return x + y; }
 static inline float  s_sub_32f(float x, float y) { return x - y; }
 static inline float s_subr_32f(float x, float y) { return y - x; }
 
@@ -946,11 +942,6 @@ static void run_arithm_s1(uchar out[], const float in[], int width, const float
     }
 }
 
-static void run_arithm_s_add3(uchar out[], const uchar in[], int width, const uchar scalar[])
-{
-    run_arithm_s3(out, in, width, scalar, v_add_16u, s_add_8u);
-}
-
 static void run_arithm_s_sub3(uchar out[], const uchar in[], int width, const uchar scalar[])
 {
     run_arithm_s3(out, in, width, scalar, v_sub_16u, s_sub_8u);
@@ -961,11 +952,6 @@ static void run_arithm_s_subr3(uchar out[], const uchar in[], int width, const u
     run_arithm_s3(out, in, width, scalar, v_subr_16u, s_subr_8u); // reverse: subr
 }
 
-static void run_arithm_s_add1(uchar out[], const float in[], int width, const float scalar[])
-{
-    run_arithm_s1(out, in, width, scalar, v_add_32f, s_add_32f);
-}
-
 static void run_arithm_s_sub1(uchar out[], const float in[], int width, const float scalar[])
 {
     run_arithm_s1(out, in, width, scalar, v_sub_32f, s_sub_32f);
@@ -1279,57 +1265,54 @@ static void run_absdiffc(Buffer &dst, const View &src, const float scalar[])
 }
 
 template<typename DST, typename SRC>
-static void run_arithm_s(Buffer &dst, const View &src, const float scalar[4], Arithm arithm,
-                         float scale=1)
+CV_ALWAYS_INLINE void run_arithm_s(Buffer &dst, const View &src, const float scalar[],
+                                   Arithm arithm, float scale=1)
 {
     const auto *in  = src.InLine<SRC>(0);
           auto *out = dst.OutLine<DST>();
 
     int width  = dst.length();
     int chan   = dst.meta().chan;
 
-    // What if we cast the scalar into the SRC type?
-    const SRC myscal[4] = { static_cast<SRC>(scalar[0]), static_cast<SRC>(scalar[1]),
-                            static_cast<SRC>(scalar[2]), static_cast<SRC>(scalar[3]) };
-    bool usemyscal = (myscal[0] == scalar[0]) && (myscal[1] == scalar[1]) &&
-                     (myscal[2] == scalar[2]) && (myscal[3] == scalar[3]);
-
     switch (arithm)
     {
     case ARITHM_ADD:
-        if (usemyscal)
-        {
-            if (std::is_same<DST,uchar>::value &&
-                std::is_same<SRC,uchar>::value &&
-                chan == 3)
-                run_arithm_s_add3((uchar*)out, (const uchar*)in, width, (const uchar*)myscal);
-            else if (std::is_same<DST,uchar>::value &&
-                     std::is_same<SRC,float>::value &&
-                     chan == 1)
-                run_arithm_s_add1((uchar*)out, (const float*)in, width, (const float*)myscal);
-            else
-                run_arithm_s(out, in, width, chan, myscal, add<DST,SRC,SRC>);
-        }
-        else
-            run_arithm_s(out, in, width, chan, scalar, add<DST,SRC,float>);
+    {
+            int w = 0;
+#if CV_SIMD
+            w = addc_simd(in, scalar, out, width, chan);
+#endif
+
+            for (; w < width * chan; ++w)
+                out[w] = add<DST>(in[w], scalar[w % chan]);
+
         break;
+    }
     case ARITHM_SUBTRACT:
+    {
+        // What if we cast the scalar into the SRC type?
+        const SRC myscal[4] = { static_cast<SRC>(scalar[0]), static_cast<SRC>(scalar[1]),
+                                static_cast<SRC>(scalar[2]), static_cast<SRC>(scalar[3]) };
+        bool usemyscal = (myscal[0] == scalar[0]) && (myscal[1] == scalar[1]) &&
+            (myscal[2] == scalar[2]) && (myscal[3] == scalar[3]);
+
         if (usemyscal)
         {
-            if (std::is_same<DST,uchar>::value &&
-                std::is_same<SRC,uchar>::value &&
+            if (std::is_same<DST, uchar>::value &&
+                std::is_same<SRC, uchar>::value &&
                 chan == 3)
                 run_arithm_s_sub3((uchar*)out, (const uchar*)in, width, (const uchar*)myscal);
-            else if (std::is_same<DST,uchar>::value &&
-                     std::is_same<SRC,float>::value &&
-                     chan == 1)
+            else if (std::is_same<DST, uchar>::value &&
+                std::is_same<SRC, float>::value &&
+                chan == 1)
                 run_arithm_s_sub1((uchar*)out, (const float*)in, width, (const float*)myscal);
             else
-                run_arithm_s(out, in, width, chan, myscal, sub<DST,SRC,SRC>);
+                run_arithm_s(out, in, width, chan, myscal, sub<DST, SRC, SRC>);
         }
         else
-            run_arithm_s(out, in, width, chan, scalar, sub<DST,SRC,float>);
+            run_arithm_s(out, in, width, chan, scalar, sub<DST, SRC, float>);
         break;
+    }
     // TODO: optimize miltiplication and division
     case ARITHM_MULTIPLY:
         for (int w=0; w < width; w++)
@@ -1433,30 +1416,75 @@ GAPI_FLUID_KERNEL(GFluidAbsDiffC, cv::gapi::core::GAbsDiffC, true)
     }
 };
 
-GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, false)
+GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, true)
 {
     static const int Window = 1;
 
-    static void run(const View &src, const cv::Scalar &_scalar, int /*dtype*/, Buffer &dst)
+    static void run(const View &src, const cv::Scalar &_scalar, int /*dtype*/, Buffer &dst, Buffer &scratch)
     {
-        const float scalar[4] = {
-            static_cast<float>(_scalar[0]),
-            static_cast<float>(_scalar[1]),
-            static_cast<float>(_scalar[2]),
-            static_cast<float>(_scalar[3])
-        };
+        GAPI_Assert(src.meta().chan <= 4);
+
+        if (dst.y() == 0)
+        {
+            const int chan = src.meta().chan;
+            float* sc = scratch.OutLine<float>();
+
+            for (int i = 0; i < scratch.length(); ++i)
+                sc[i] = static_cast<float>(_scalar[i % chan]);
+        }
+
+        const float* scalar = scratch.OutLine<float>();
 
         //     DST     SRC     OP            __VA_ARGS__
-        UNARY_(uchar , uchar , run_arithm_s, dst, src, scalar, ARITHM_ADD);
-        UNARY_(uchar ,  short, run_arithm_s, dst, src, scalar, ARITHM_ADD);
-        UNARY_(uchar ,  float, run_arithm_s, dst, src, scalar, ARITHM_ADD);
-        UNARY_( short,  short, run_arithm_s, dst, src, scalar, ARITHM_ADD);
-        UNARY_( float, uchar , run_arithm_s, dst, src, scalar, ARITHM_ADD);
-        UNARY_( float,  short, run_arithm_s, dst, src, scalar, ARITHM_ADD);
-        UNARY_( float,  float, run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(uchar,  uchar,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(uchar,  ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(uchar,  short,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(uchar,  float,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(ushort, ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(ushort, short,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(ushort, uchar,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(ushort, float,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(short,  short,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(short,  ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(short,  uchar,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(short,  float,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(float,  uchar,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(float,  ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(float,  short,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
+        UNARY_(float,  float,  run_arithm_s, dst, src, scalar, ARITHM_ADD);
 
         CV_Error(cv::Error::StsBadArg, "unsupported combination of types");
     }
+
+    static void initScratch(const GMatDesc&, const GScalarDesc&, int, Buffer& scratch)
+    {
+#if CV_SIMD
+        // 512 bits / 32 bits = 16 elements of float32 can contain a AVX 512 SIMD vector.
+        constexpr int maxNlanes = 16;
+
+        // +2 is offset for 3-channel case.
+        // Offset is need to right load coefficients from scalar array to SIMD vectors for 3-channel case.
+        // Scalar array looks like: scalar[] = {C1, C2, C3, C1, C2, C3, ...}
+        // The first scalar SIMD vector should looks like:
+        // C1 C2 C3 C1
+        // The second:
+        // C2 C3 C1 C2
+        // The third:
+        // C3 C1 C2 C3
+        constexpr int offset = 2;
+        constexpr int buflen = maxNlanes + offset;
+#else
+        constexpr int buflen = 4;
+#endif
+        cv::Size bufsize(buflen, 1);
+        GMatDesc bufdesc = { CV_32F, 1, bufsize };
+        Buffer buffer(bufdesc);
+        scratch = std::move(buffer);
+    }
+
+    static void resetScratch(Buffer& /* scratch */)
+    {
+    }
 };
 
 GAPI_FLUID_KERNEL(GFluidSubC, cv::gapi::core::GSubC, false)
diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp
@@ -85,6 +85,33 @@ MUL_SIMD(float, float)
 
 #undef MUL_SIMD
 
+#define ADDC_SIMD(SRC, DST)                                               \
+int addc_simd(const SRC in[], const float scalar[], DST out[],            \
+              const int width, const int chan)                            \
+{                                                                         \
+    CV_CPU_DISPATCH(addc_simd, (in, scalar, out, width, chan),            \
+                    CV_CPU_DISPATCH_MODES_ALL);                           \
+}
+
+ADDC_SIMD(uchar, uchar)
+ADDC_SIMD(ushort, uchar)
+ADDC_SIMD(short, uchar)
+ADDC_SIMD(float, uchar)
+ADDC_SIMD(short, short)
+ADDC_SIMD(ushort, short)
+ADDC_SIMD(uchar, short)
+ADDC_SIMD(float, short)
+ADDC_SIMD(ushort, ushort)
+ADDC_SIMD(uchar, ushort)
+ADDC_SIMD(short, ushort)
+ADDC_SIMD(float, ushort)
+ADDC_SIMD(uchar, float)
+ADDC_SIMD(ushort, float)
+ADDC_SIMD(short, float)
+ADDC_SIMD(float, float)
+
+#undef ADDC_SIMD
+
 } // namespace fluid
 } // namespace gapi
 } // namespace cv
diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp
@@ -60,6 +60,29 @@ MUL_SIMD(float, float)
 
 #undef MUL_SIMD
 
+#define ADDC_SIMD(SRC, DST)                                                              \
+int addc_simd(const SRC in[], const float scalar[], DST out[],                           \
+              const int width, const int chan);
+
+ADDC_SIMD(uchar, uchar)
+ADDC_SIMD(ushort, uchar)
+ADDC_SIMD(short, uchar)
+ADDC_SIMD(float, uchar)
+ADDC_SIMD(short, short)
+ADDC_SIMD(ushort, short)
+ADDC_SIMD(uchar, short)
+ADDC_SIMD(float, short)
+ADDC_SIMD(ushort, ushort)
+ADDC_SIMD(uchar, ushort)
+ADDC_SIMD(short, ushort)
+ADDC_SIMD(float, ushort)
+ADDC_SIMD(uchar, float)
+ADDC_SIMD(ushort, float)
+ADDC_SIMD(short, float)
+ADDC_SIMD(float, float)
+
+#undef ADDC_SIMD
+
 }  // namespace fluid
 }  // namespace gapi
 }  // namespace cv
diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp

Original file line number	Diff line number	Diff line change
`@@ -57,6 +57,7 @@ namespace core {`
`57`	`57`
`58`	`58`	`G_TYPED_KERNEL(GAddC, <GMat(GMat, GScalar, int)>, "org.opencv.core.math.addC") {`
`59`	`59`	`static GMatDesc outMeta(GMatDesc a, GScalarDesc, int ddepth) {`
	`60`	`+ GAPI_Assert(a.chan <= 4);`
`60`	`61`	`return a.withDepth(ddepth);`
`61`	`62`	`}`
`62`	`63`	`};`