mirror of
https://github.com/opencv/opencv.git
synced 2026-09-25 04:09:57 +03:00
Merge pull request #29953 from pranayr710:feat/texpr-fuse-addweighted
core: fold a*alpha + b*beta + gamma into the fused OP_ADDW kernel - #29953
Part of #29443 — "fuse `a*alpha + b*beta + gamma`, where alpha, beta and gamma are scalars, into
some new `OP_ADDW`". Builds on #29937, which is required for correctness (see below).
### Problem
`OP_ADDW` already computes `a*alpha + b*beta + gamma` as a single kernel over two `v_fma`, and
`emitBinary()` already knows how to emit it — but the string front-end never recognized the
pattern, so `cv::texpr()` always took the written-out path: two multiplies, two adds, three temp
buffers and four passes over the data.
```
{0}*2.0 + {1}*3.0 + 1.0 at CV_32F
insns=4 temps=3 buffers=3 insns=1 temps=0 buffers=0
0: mul(1, 4) -> 5 ==> 0: addWeighted(1, 2) -> 11
1: mul(2, 7) -> 8 params=[2, 3, 1]
2: add(5, 8) -> 9
3: add(9, 11) -> 13
```
### Fix
A peephole in `emitBinary()`: `a*alpha + b*beta` folds into one `OP_ADDW`, and a trailing scalar
folds into that instruction's gamma rather than costing another pass.
`emitBinary()` may wrap a multiply in casts — an integer array times a fractional scalar computes
in the float domain and lands back in the array's own type — so the matcher accepts the optional
widening and narrowing casts around it. That is what makes the common 8-bit blend fuse; without
it `{0}*0.7 + {1}*0.3` at `CV_8U` stays at eight instructions and seven temps.
Shapes that are not an addWeighted keep their own meaning: an `a*b` term has no scalar factor, a
per-channel constant cannot ride the params block, and `CV_Bool` has no `OP_ADDW` form.
**Dependency on #29937.** This retires the instructions it folds, exactly as the
`abs(x - y) -> absdiff` peephole does. Without the `pinned` flag added in #29937 it would
reintroduce that bug for `u = {0}*2.0; v = {1}*3.0; u + v`, where the named terms are still live.
### Semantics on integer types
The fused kernel evaluates at its own work precision, so intermediate results no longer saturate
at each step. On integer inputs the result changes — it now agrees with `cv::addWeighted`, which
is what the expression means. This is the same trade the existing `abs(x - y)` peephole documents
in `emitUnary()`: the saturation artifacts of the literal expansion are never the desired result.
### Performance
1920x1080, best of 5 runs of 50 iterations, same build rebuilt both ways on 03ae9eac50:
| expression | depth | before | after |
| --- | --- | --- | --- |
| `a*0.7 + b*0.3` | 8U | 0.646 ms | 0.159 ms |
| `a*2 + b*3 + 1` | 8U | 0.261 ms | 0.150 ms |
| `a*2.5 + b*-1.5 + 7` | 32F | 0.360 ms | 0.214 ms |
| `a*2 + b*3 + 1` | 32F | 0.213 ms | 0.155 ms |
| `a*b + b*2` (control, not fused) | 32F | 0.393 ms | 0.358 ms |
The control row runs identical code in both builds and still moves by ~9%, so run-to-run noise on
this machine is around 10% — treat the 32F rows as indicative and the 8-bit rows as the real
result. Timings include `cv::texpr()` re-parsing and recompiling the expression on every call, so
the kernel-level gain is larger than the totals suggest.
### Tests
28 parameterised cases — 7 depths crossed with 4 weight sets, including a zero gamma and negative
weights — assert the fused result matches `cv::addWeighted`. Two further tests cover the variants
(no gamma, scalar written first, leading gamma) and the shapes the peephole must decline. 1801
tests in the arithmetic and TExpr suites pass.
Also adds the missing `OP_ADDW` case to `opName()`, which this change makes visible in every dump
of such a program.
### Pull Request Readiness Checklist
See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request
- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
@@ -36,4 +36,32 @@ PERF_TEST_P(Size_MatType, addWeighted, TYPICAL_MATS_ADWEIGHTED)
|
||||
SANITY_CHECK_NOTHING();
|
||||
}
|
||||
|
||||
|
||||
// The same computation written as an expression. cv::texpr() folds a*alpha + b*beta + gamma into
|
||||
// the single fused OP_ADDW kernel instead of emitting two multiplies, two adds and three temp
|
||||
// buffers, so this shares the grid above and can be compared against it directly.
|
||||
PERF_TEST_P(Size_MatType, texpr_addWeighted, TYPICAL_MATS_ADWEIGHTED)
|
||||
{
|
||||
Size size = get<0>(GetParam());
|
||||
int type = get<1>(GetParam());
|
||||
int depth = CV_MAT_DEPTH(type);
|
||||
Mat src1(size, type);
|
||||
Mat src2(size, type);
|
||||
|
||||
declare.in(src1, src2, WARMUP_RNG);
|
||||
|
||||
if (depth == CV_32S)
|
||||
{
|
||||
// there might be not enough precision for integers
|
||||
src1 /= 2048;
|
||||
src2 /= 2048;
|
||||
}
|
||||
|
||||
std::vector<Mat> inputs{ src1, src2 }, outputs;
|
||||
|
||||
TEST_CYCLE() cv::texpr("{0} * 3.75 + {1} * -0.125 + 100.0", inputs, outputs);
|
||||
|
||||
SANITY_CHECK_NOTHING();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -48,6 +48,7 @@ const char* opName(TOp op)
|
||||
case OP_MIN: return "min";
|
||||
case OP_MAX: return "max";
|
||||
case OP_ABSDIFF: return "absdiff";
|
||||
case OP_ADDW: return "addWeighted";
|
||||
case OP_HYPOT: return "hypot";
|
||||
case OP_ATAN2: return "atan2";
|
||||
case OP_AND: return "and";
|
||||
@@ -294,6 +295,63 @@ static inline bool isFlexConst(const TExpr& e, int s)
|
||||
return e.arginfo[s].kind == TExpr::CONST && e.arginfo[s].depth == EW_DEPTH_NONE;
|
||||
}
|
||||
|
||||
// Read a CONST slot that carries exactly ONE value. OP_ADDW transports alpha/beta/gamma in the
|
||||
// instruction's params block - a single value applied to every channel - so a per-channel const
|
||||
// cannot ride it and must keep the expanded form.
|
||||
static bool scalarConstValue(const TExpr& e, int s, double& v)
|
||||
{
|
||||
const TExpr::Arg& a = e.arginfo[s];
|
||||
if (a.kind != TExpr::CONST || std::max(1, a.channels) != 1) return false;
|
||||
AutoBuffer<double, MAX_LOCAL_CN> buf;
|
||||
constDoubles(e, s, buf);
|
||||
v = buf[0];
|
||||
return true;
|
||||
}
|
||||
|
||||
// Does the run of instructions ENDING at `end` compute `array * scalar` into slot `t`?
|
||||
//
|
||||
// emitBinary() may wrap the multiply in casts: an integer array times a fractional scalar computes
|
||||
// in the float domain and lands back in the array's own type, so the run is
|
||||
// [cast widen] -> mul -> [cast narrow]
|
||||
// with the multiply always present and the two casts optional. All three are contiguous and sit at
|
||||
// the tail of the program (the operand was emitted immediately before the pending add), so matching
|
||||
// backwards from `end` identifies the whole run. Fills the ORIGINAL array operand, the scalar, and
|
||||
// the index of the run's first instruction.
|
||||
static bool matchScaledChain(const TExpr& e, int end, int t,
|
||||
int& arrSlot, double& scale, int& startIdx)
|
||||
{
|
||||
if (end < 0) return false;
|
||||
int j = end, mulResult = t;
|
||||
|
||||
// optional trailing narrowing cast
|
||||
if (e.prog[j].op == OP_CAST && e.prog[j].result == t)
|
||||
{
|
||||
mulResult = e.prog[j].arg0;
|
||||
if (--j < 0) return false;
|
||||
}
|
||||
|
||||
const TExpr::Insn& mul = e.prog[j];
|
||||
if (mul.op != OP_MUL || mul.result != mulResult || mul.params[0] != 1.0) return false;
|
||||
|
||||
int arr;
|
||||
if (scalarConstValue(e, mul.arg1, scale)) arr = mul.arg0;
|
||||
else if (scalarConstValue(e, mul.arg0, scale)) arr = mul.arg1;
|
||||
else return false; // array*array is not an addWeighted term
|
||||
startIdx = j;
|
||||
|
||||
// optional leading widening cast feeding the multiply
|
||||
if (j - 1 >= 0 && e.prog[j-1].op == OP_CAST && e.prog[j-1].result == arr &&
|
||||
e.arginfo[arr].kind == TExpr::TEMP)
|
||||
{
|
||||
arr = e.prog[j-1].arg0;
|
||||
startIdx = j - 1;
|
||||
}
|
||||
|
||||
if (e.arginfo[arr].kind == TExpr::CONST) return false; // scalar*scalar is folded elsewhere
|
||||
arrSlot = arr;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Can a flexible CONST `s` be represented exactly at depth `d`? (typed operands trivially "fit").
|
||||
static bool constFits(const TExpr& e, int s, int d)
|
||||
{
|
||||
@@ -345,6 +403,68 @@ int TExpr::emitBinary(TOp op, int a, int b, int rdepth, const Scalar& params)
|
||||
return out2;
|
||||
}
|
||||
|
||||
// peephole: a*alpha + b*beta [+ gamma] -> the fused OP_ADDW kernel (two v_fma). Written out,
|
||||
// the form costs four instructions, three temp buffers and four passes over the data; OP_ADDW
|
||||
// does the whole thing in one pass with no temp. Like the abs(x - y) -> absdiff peephole in
|
||||
// emitUnary() this RETIRES the instructions it folds, so - same as there - it must leave a
|
||||
// pinned (named) slot alone: see TExpr::Arg::pinned.
|
||||
if (op == OP_ADD)
|
||||
{
|
||||
const int n = (int)prog.size();
|
||||
|
||||
// (x*alpha) + (y*beta). Both multiplies are the last two instructions and their results the
|
||||
// last two temps (the operands were emitted immediately before this add), so folding them is
|
||||
// a pop of the program tail and no slot renumbering is needed.
|
||||
int x = 0, y = 0, startA = 0, startB = 0;
|
||||
double alpha = 0, beta = 0;
|
||||
if (n >= 2 && a != b &&
|
||||
arginfo[a].kind == TEMP && !arginfo[a].pinned &&
|
||||
arginfo[b].kind == TEMP && !arginfo[b].pinned &&
|
||||
matchScaledChain(*this, n - 1, b, y, beta, startB) &&
|
||||
matchScaledChain(*this, startB - 1, a, x, alpha, startA) &&
|
||||
x != a && x != b && y != a && y != b &&
|
||||
// OP_ADDW has no CV_Bool form (it would have to pick a numeric work type); leave a bool
|
||||
// operand to the plain expansion rather than turning it into an error.
|
||||
arginfo[x].depth != CV_Bool && arginfo[y].depth != CV_Bool)
|
||||
{
|
||||
// The two runs together are exactly the tail [startA, n), so folding them is a truncation
|
||||
// of the program; every temp they produced was created after every surviving one, so
|
||||
// retiring them keeps the temp indices dense (0..ntemps-1) for compile()'s liveness.
|
||||
int retired = 0, lowest = INT_MAX;
|
||||
for (int i = startA; i < n; i++)
|
||||
{
|
||||
const int r = prog[i].result;
|
||||
if (arginfo[r].kind != TEMP || arginfo[r].pinned) { retired = -1; break; }
|
||||
lowest = std::min(lowest, arginfo[r].index);
|
||||
retired++;
|
||||
}
|
||||
if (retired > 0 && lowest == ntemps - retired)
|
||||
{
|
||||
for (int i = startA; i < n; i++) arginfo[prog[i].result].kind = NONE;
|
||||
prog.resize(startA);
|
||||
ntemps -= retired;
|
||||
return emitBinary(OP_ADDW, x, y, rdepth, Scalar(alpha, beta, 0.));
|
||||
}
|
||||
}
|
||||
|
||||
// (x*alpha + y*beta) + gamma: the ADDW is already emitted and its gamma still 0, so the
|
||||
// trailing scalar folds into the instruction's params instead of costing another pass.
|
||||
{
|
||||
int acc = 0; double gamma = 0;
|
||||
if (scalarConstValue(*this, b, gamma)) acc = a;
|
||||
else if (scalarConstValue(*this, a, gamma)) acc = b;
|
||||
if (acc != 0 && n >= 1 &&
|
||||
arginfo[acc].kind == TEMP && !arginfo[acc].pinned &&
|
||||
arginfo[acc].index == ntemps - 1 &&
|
||||
prog[n-1].op == OP_ADDW && prog[n-1].result == acc &&
|
||||
(rdepth == EW_DEPTH_NONE || rdepth == arginfo[acc].depth))
|
||||
{
|
||||
prog[n-1].params[2] += gamma;
|
||||
return acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const int nd0 = isFlexConst(*this, a) ? EW_DEPTH_NONE : arginfo[a].depth;
|
||||
const int nd1 = isFlexConst(*this, b) ? EW_DEPTH_NONE : arginfo[b].depth;
|
||||
const ElemwiseCategory cat = opCategory(op);
|
||||
|
||||
@@ -573,4 +573,73 @@ TEST(Core_TExpr, named_value_single_use_still_correct)
|
||||
EXPECT_LE(cvtest::norm(got2, exp2, NORM_INF), 1e-3);
|
||||
}
|
||||
|
||||
|
||||
// a*alpha + b*beta [+ gamma] is folded into the single fused OP_ADDW kernel. The fused form
|
||||
// evaluates the whole expression at the kernel's own work precision, so on integer types it does
|
||||
// NOT saturate at each intermediate step the way the written-out multiplies do - it agrees with
|
||||
// cv::addWeighted, which is what the expression means. These check exactly that agreement.
|
||||
typedef testing::TestWithParam< tuple<int, int> > Core_TExpr_AddW;
|
||||
|
||||
TEST_P(Core_TExpr_AddW, matches_addWeighted)
|
||||
{
|
||||
const int depth = get<0>(GetParam());
|
||||
const int which = get<1>(GetParam());
|
||||
static const double A[] = { 2.0, 2.5, -1.5, 0.25 };
|
||||
static const double B[] = { 3.0, -1.5, 0.5, 0.75 };
|
||||
static const double G[] = { 1.0, 7.0, 0.0, -3.5 };
|
||||
const double alpha = A[which], beta = B[which], gamma = G[which];
|
||||
|
||||
Mat a(17, 23, depth), b(17, 23, depth);
|
||||
theRNG().fill(a, RNG::UNIFORM, 0, 50);
|
||||
theRNG().fill(b, RNG::UNIFORM, 0, 50);
|
||||
|
||||
const String e = cv::format("{0} * %.17g + {1} * %.17g + %.17g", alpha, beta, gamma);
|
||||
Mat got = expr1(e, { a, b });
|
||||
|
||||
Mat exp; cv::addWeighted(a, alpha, b, beta, gamma, exp);
|
||||
ASSERT_EQ(exp.type(), got.type());
|
||||
EXPECT_LE(cvtest::norm(got, exp, NORM_INF), 1e-3) << e << " on depth " << depth;
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(/**/, Core_TExpr_AddW,
|
||||
testing::Combine(testing::Values(CV_8U, CV_8S, CV_16U, CV_16S, CV_32S, CV_32F, CV_64F),
|
||||
testing::Values(0, 1, 2, 3)));
|
||||
|
||||
// gamma may be absent, and either operand order of the scalar is the same expression.
|
||||
TEST(Core_TExpr, addweighted_fusion_variants)
|
||||
{
|
||||
Mat a(12, 15, CV_32F), b(12, 15, CV_32F);
|
||||
theRNG().fill(a, RNG::UNIFORM, 1.f, 10.f);
|
||||
theRNG().fill(b, RNG::UNIFORM, 1.f, 10.f);
|
||||
|
||||
Mat exp; cv::addWeighted(a, 2.0, b, 3.0, 0.0, exp);
|
||||
EXPECT_LE(cvtest::norm(expr1("{0}*2.0 + {1}*3.0", { a, b }), exp, NORM_INF), 1e-3) << "no gamma";
|
||||
EXPECT_LE(cvtest::norm(expr1("2.0*{0} + 3.0*{1}", { a, b }), exp, NORM_INF), 1e-3) << "scalar first";
|
||||
|
||||
Mat expg; cv::addWeighted(a, 2.0, b, 3.0, 5.0, expg);
|
||||
EXPECT_LE(cvtest::norm(expr1("5.0 + {0}*2.0 + {1}*3.0", { a, b }), expg, NORM_INF), 1e-3) << "leading gamma";
|
||||
}
|
||||
|
||||
// Shapes that look similar but are NOT an addWeighted must keep their own meaning.
|
||||
TEST(Core_TExpr, addweighted_fusion_declined)
|
||||
{
|
||||
Mat a(10, 14, CV_32F), b(10, 14, CV_32F);
|
||||
theRNG().fill(a, RNG::UNIFORM, 1.f, 10.f);
|
||||
theRNG().fill(b, RNG::UNIFORM, 1.f, 10.f);
|
||||
|
||||
// a*b is not a scaled input - the first term has no scalar factor.
|
||||
Mat got = expr1("{0}*{1} + {1}*2.0", { a, b });
|
||||
Mat ab, b2, exp; cv::multiply(a, b, ab); cv::multiply(b, 2.0, b2); cv::add(ab, b2, exp);
|
||||
EXPECT_LE(cvtest::norm(got, exp, NORM_INF), 1e-3) << "array * array term";
|
||||
|
||||
// A named term is used again below, so its multiply cannot be folded away.
|
||||
std::vector<Mat> out;
|
||||
cv::texpr("u = {0}*2.0; v = {1}*3.0; (u + v, u)", std::vector<Mat>{ a, b }, out);
|
||||
ASSERT_EQ(out.size(), 2u);
|
||||
Mat sum; cv::addWeighted(a, 2.0, b, 3.0, 0.0, sum);
|
||||
Mat u; cv::multiply(a, 2.0, u);
|
||||
EXPECT_LE(cvtest::norm(out[0], sum, NORM_INF), 1e-3) << "u + v";
|
||||
EXPECT_LE(cvtest::norm(out[1], u, NORM_INF), 1e-3) << "u reused";
|
||||
}
|
||||
|
||||
}} // namespace
|
||||
|
||||
Reference in New Issue
Block a user