diff --git a/modules/core/perf/perf_addWeighted.cpp b/modules/core/perf/perf_addWeighted.cpp index a0639ac6b0..3824d16e50 100644 --- a/modules/core/perf/perf_addWeighted.cpp +++ b/modules/core/perf/perf_addWeighted.cpp @@ -36,4 +36,32 @@ PERF_TEST_P(Size_MatType, addWeighted, TYPICAL_MATS_ADWEIGHTED) SANITY_CHECK_NOTHING(); } + +// The same computation written as an expression. cv::texpr() folds a*alpha + b*beta + gamma into +// the single fused OP_ADDW kernel instead of emitting two multiplies, two adds and three temp +// buffers, so this shares the grid above and can be compared against it directly. +PERF_TEST_P(Size_MatType, texpr_addWeighted, TYPICAL_MATS_ADWEIGHTED) +{ + Size size = get<0>(GetParam()); + int type = get<1>(GetParam()); + int depth = CV_MAT_DEPTH(type); + Mat src1(size, type); + Mat src2(size, type); + + declare.in(src1, src2, WARMUP_RNG); + + if (depth == CV_32S) + { + // there might be not enough precision for integers + src1 /= 2048; + src2 /= 2048; + } + + std::vector inputs{ src1, src2 }, outputs; + + TEST_CYCLE() cv::texpr("{0} * 3.75 + {1} * -0.125 + 100.0", inputs, outputs); + + SANITY_CHECK_NOTHING(); +} + } // namespace diff --git a/modules/core/src/arithm_expr.cpp b/modules/core/src/arithm_expr.cpp index 5c09c38669..e35505b96c 100644 --- a/modules/core/src/arithm_expr.cpp +++ b/modules/core/src/arithm_expr.cpp @@ -48,6 +48,7 @@ const char* opName(TOp op) case OP_MIN: return "min"; case OP_MAX: return "max"; case OP_ABSDIFF: return "absdiff"; + case OP_ADDW: return "addWeighted"; case OP_HYPOT: return "hypot"; case OP_ATAN2: return "atan2"; case OP_AND: return "and"; @@ -294,6 +295,63 @@ static inline bool isFlexConst(const TExpr& e, int s) return e.arginfo[s].kind == TExpr::CONST && e.arginfo[s].depth == EW_DEPTH_NONE; } +// Read a CONST slot that carries exactly ONE value. OP_ADDW transports alpha/beta/gamma in the +// instruction's params block - a single value applied to every channel - so a per-channel const +// cannot ride it and must keep the expanded form. +static bool scalarConstValue(const TExpr& e, int s, double& v) +{ + const TExpr::Arg& a = e.arginfo[s]; + if (a.kind != TExpr::CONST || std::max(1, a.channels) != 1) return false; + AutoBuffer buf; + constDoubles(e, s, buf); + v = buf[0]; + return true; +} + +// Does the run of instructions ENDING at `end` compute `array * scalar` into slot `t`? +// +// emitBinary() may wrap the multiply in casts: an integer array times a fractional scalar computes +// in the float domain and lands back in the array's own type, so the run is +// [cast widen] -> mul -> [cast narrow] +// with the multiply always present and the two casts optional. All three are contiguous and sit at +// the tail of the program (the operand was emitted immediately before the pending add), so matching +// backwards from `end` identifies the whole run. Fills the ORIGINAL array operand, the scalar, and +// the index of the run's first instruction. +static bool matchScaledChain(const TExpr& e, int end, int t, + int& arrSlot, double& scale, int& startIdx) +{ + if (end < 0) return false; + int j = end, mulResult = t; + + // optional trailing narrowing cast + if (e.prog[j].op == OP_CAST && e.prog[j].result == t) + { + mulResult = e.prog[j].arg0; + if (--j < 0) return false; + } + + const TExpr::Insn& mul = e.prog[j]; + if (mul.op != OP_MUL || mul.result != mulResult || mul.params[0] != 1.0) return false; + + int arr; + if (scalarConstValue(e, mul.arg1, scale)) arr = mul.arg0; + else if (scalarConstValue(e, mul.arg0, scale)) arr = mul.arg1; + else return false; // array*array is not an addWeighted term + startIdx = j; + + // optional leading widening cast feeding the multiply + if (j - 1 >= 0 && e.prog[j-1].op == OP_CAST && e.prog[j-1].result == arr && + e.arginfo[arr].kind == TExpr::TEMP) + { + arr = e.prog[j-1].arg0; + startIdx = j - 1; + } + + if (e.arginfo[arr].kind == TExpr::CONST) return false; // scalar*scalar is folded elsewhere + arrSlot = arr; + return true; +} + // Can a flexible CONST `s` be represented exactly at depth `d`? (typed operands trivially "fit"). static bool constFits(const TExpr& e, int s, int d) { @@ -345,6 +403,68 @@ int TExpr::emitBinary(TOp op, int a, int b, int rdepth, const Scalar& params) return out2; } + // peephole: a*alpha + b*beta [+ gamma] -> the fused OP_ADDW kernel (two v_fma). Written out, + // the form costs four instructions, three temp buffers and four passes over the data; OP_ADDW + // does the whole thing in one pass with no temp. Like the abs(x - y) -> absdiff peephole in + // emitUnary() this RETIRES the instructions it folds, so - same as there - it must leave a + // pinned (named) slot alone: see TExpr::Arg::pinned. + if (op == OP_ADD) + { + const int n = (int)prog.size(); + + // (x*alpha) + (y*beta). Both multiplies are the last two instructions and their results the + // last two temps (the operands were emitted immediately before this add), so folding them is + // a pop of the program tail and no slot renumbering is needed. + int x = 0, y = 0, startA = 0, startB = 0; + double alpha = 0, beta = 0; + if (n >= 2 && a != b && + arginfo[a].kind == TEMP && !arginfo[a].pinned && + arginfo[b].kind == TEMP && !arginfo[b].pinned && + matchScaledChain(*this, n - 1, b, y, beta, startB) && + matchScaledChain(*this, startB - 1, a, x, alpha, startA) && + x != a && x != b && y != a && y != b && + // OP_ADDW has no CV_Bool form (it would have to pick a numeric work type); leave a bool + // operand to the plain expansion rather than turning it into an error. + arginfo[x].depth != CV_Bool && arginfo[y].depth != CV_Bool) + { + // The two runs together are exactly the tail [startA, n), so folding them is a truncation + // of the program; every temp they produced was created after every surviving one, so + // retiring them keeps the temp indices dense (0..ntemps-1) for compile()'s liveness. + int retired = 0, lowest = INT_MAX; + for (int i = startA; i < n; i++) + { + const int r = prog[i].result; + if (arginfo[r].kind != TEMP || arginfo[r].pinned) { retired = -1; break; } + lowest = std::min(lowest, arginfo[r].index); + retired++; + } + if (retired > 0 && lowest == ntemps - retired) + { + for (int i = startA; i < n; i++) arginfo[prog[i].result].kind = NONE; + prog.resize(startA); + ntemps -= retired; + return emitBinary(OP_ADDW, x, y, rdepth, Scalar(alpha, beta, 0.)); + } + } + + // (x*alpha + y*beta) + gamma: the ADDW is already emitted and its gamma still 0, so the + // trailing scalar folds into the instruction's params instead of costing another pass. + { + int acc = 0; double gamma = 0; + if (scalarConstValue(*this, b, gamma)) acc = a; + else if (scalarConstValue(*this, a, gamma)) acc = b; + if (acc != 0 && n >= 1 && + arginfo[acc].kind == TEMP && !arginfo[acc].pinned && + arginfo[acc].index == ntemps - 1 && + prog[n-1].op == OP_ADDW && prog[n-1].result == acc && + (rdepth == EW_DEPTH_NONE || rdepth == arginfo[acc].depth)) + { + prog[n-1].params[2] += gamma; + return acc; + } + } + } + const int nd0 = isFlexConst(*this, a) ? EW_DEPTH_NONE : arginfo[a].depth; const int nd1 = isFlexConst(*this, b) ? EW_DEPTH_NONE : arginfo[b].depth; const ElemwiseCategory cat = opCategory(op); diff --git a/modules/core/test/test_arithm_expr.cpp b/modules/core/test/test_arithm_expr.cpp index 7837f69b67..d638964f16 100644 --- a/modules/core/test/test_arithm_expr.cpp +++ b/modules/core/test/test_arithm_expr.cpp @@ -573,4 +573,73 @@ TEST(Core_TExpr, named_value_single_use_still_correct) EXPECT_LE(cvtest::norm(got2, exp2, NORM_INF), 1e-3); } + +// a*alpha + b*beta [+ gamma] is folded into the single fused OP_ADDW kernel. The fused form +// evaluates the whole expression at the kernel's own work precision, so on integer types it does +// NOT saturate at each intermediate step the way the written-out multiplies do - it agrees with +// cv::addWeighted, which is what the expression means. These check exactly that agreement. +typedef testing::TestWithParam< tuple > Core_TExpr_AddW; + +TEST_P(Core_TExpr_AddW, matches_addWeighted) +{ + const int depth = get<0>(GetParam()); + const int which = get<1>(GetParam()); + static const double A[] = { 2.0, 2.5, -1.5, 0.25 }; + static const double B[] = { 3.0, -1.5, 0.5, 0.75 }; + static const double G[] = { 1.0, 7.0, 0.0, -3.5 }; + const double alpha = A[which], beta = B[which], gamma = G[which]; + + Mat a(17, 23, depth), b(17, 23, depth); + theRNG().fill(a, RNG::UNIFORM, 0, 50); + theRNG().fill(b, RNG::UNIFORM, 0, 50); + + const String e = cv::format("{0} * %.17g + {1} * %.17g + %.17g", alpha, beta, gamma); + Mat got = expr1(e, { a, b }); + + Mat exp; cv::addWeighted(a, alpha, b, beta, gamma, exp); + ASSERT_EQ(exp.type(), got.type()); + EXPECT_LE(cvtest::norm(got, exp, NORM_INF), 1e-3) << e << " on depth " << depth; +} + +INSTANTIATE_TEST_CASE_P(/**/, Core_TExpr_AddW, + testing::Combine(testing::Values(CV_8U, CV_8S, CV_16U, CV_16S, CV_32S, CV_32F, CV_64F), + testing::Values(0, 1, 2, 3))); + +// gamma may be absent, and either operand order of the scalar is the same expression. +TEST(Core_TExpr, addweighted_fusion_variants) +{ + Mat a(12, 15, CV_32F), b(12, 15, CV_32F); + theRNG().fill(a, RNG::UNIFORM, 1.f, 10.f); + theRNG().fill(b, RNG::UNIFORM, 1.f, 10.f); + + Mat exp; cv::addWeighted(a, 2.0, b, 3.0, 0.0, exp); + EXPECT_LE(cvtest::norm(expr1("{0}*2.0 + {1}*3.0", { a, b }), exp, NORM_INF), 1e-3) << "no gamma"; + EXPECT_LE(cvtest::norm(expr1("2.0*{0} + 3.0*{1}", { a, b }), exp, NORM_INF), 1e-3) << "scalar first"; + + Mat expg; cv::addWeighted(a, 2.0, b, 3.0, 5.0, expg); + EXPECT_LE(cvtest::norm(expr1("5.0 + {0}*2.0 + {1}*3.0", { a, b }), expg, NORM_INF), 1e-3) << "leading gamma"; +} + +// Shapes that look similar but are NOT an addWeighted must keep their own meaning. +TEST(Core_TExpr, addweighted_fusion_declined) +{ + Mat a(10, 14, CV_32F), b(10, 14, CV_32F); + theRNG().fill(a, RNG::UNIFORM, 1.f, 10.f); + theRNG().fill(b, RNG::UNIFORM, 1.f, 10.f); + + // a*b is not a scaled input - the first term has no scalar factor. + Mat got = expr1("{0}*{1} + {1}*2.0", { a, b }); + Mat ab, b2, exp; cv::multiply(a, b, ab); cv::multiply(b, 2.0, b2); cv::add(ab, b2, exp); + EXPECT_LE(cvtest::norm(got, exp, NORM_INF), 1e-3) << "array * array term"; + + // A named term is used again below, so its multiply cannot be folded away. + std::vector out; + cv::texpr("u = {0}*2.0; v = {1}*3.0; (u + v, u)", std::vector{ a, b }, out); + ASSERT_EQ(out.size(), 2u); + Mat sum; cv::addWeighted(a, 2.0, b, 3.0, 0.0, sum); + Mat u; cv::multiply(a, 2.0, u); + EXPECT_LE(cvtest::norm(out[0], sum, NORM_INF), 1e-3) << "u + v"; + EXPECT_LE(cvtest::norm(out[1], u, NORM_INF), 1e-3) << "u reused"; +} + }} // namespace