mirrors/opencv
2
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-09-25 04:09:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #28741 from abhishek-gola:int8_block_layout

Browse Source 
Int8 block layout support #28741

After this patch we got the following speed ups on **resnet50-qdq.onnx** model.

- Inference time now: **_~6.6ms_** (inference time using onnxruntime is ~5.7ms).
- Inference time before: _**~11.5ms**_ [after QDQ PR #28595]
- Speed up: _**~42.6% or 1.74x**_
- Device details:
- Model name: Intel(R) Core(TM) i9-14900KS, x86, 32 Cores, ubuntu 24.04, 

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
2026-04-12 19:40:03 +03:00
committed by GitHub
parent 1a6f669763
commit 53d9a67cf3
11 changed files with 3088 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/CMakeLists.txt
+1
View File
@@ -12,6 +12,7 @@ ocv_add_dispatched_file("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON NEO
ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX)
ocv_add_dispatched_file("layers/cpu_kernels/conv2_depthwise" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file("layers/cpu_kernels/conv2_kernels" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file_force_all("int8layers/conv2_int8_kernels" AVX2)
ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js)
modules/dnn/include/opencv2/dnn/all_layers.hpp
+49
View File
@@ -382,6 +382,21 @@ CV__DNN_INLINE_NS_BEGIN
bool ceil_mode;
};
class CV_EXPORTS Conv2Int8Layer : public Layer
{
public:
static Ptr<Conv2Int8Layer> create(const LayerParams& params);
int input_zp, output_zp;
float input_sc, output_sc;
bool per_channel;
std::vector<int> strides, dilations, pads;
int ngroups;
AutoPadding auto_pad;
bool ceil_mode;
};
class CV_EXPORTS LRNLayer : public Layer
{
public:
@@ -461,6 +476,9 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<PoolingLayer> create(const LayerParams& params);
};
// Old-engine int8 pooling. Created directly by the ONNX importer for
// QLinearAveragePool / QLinearGlobalAveragePool / int8 MaxPool ops.
// Inherits PoolingLayer so it can delegate to TIMVX / NGRAPH backends.
class CV_EXPORTS PoolingLayerInt8 : public PoolingLayer
{
public:
@@ -469,6 +487,25 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<PoolingLayerInt8> create(const LayerParams& params);
};
// New-engine int8 pooling with block memory layout (DATA_LAYOUT_BLOCK).
// Created by the QDQ graph fusion pass (graph_fusion_qdq.cpp) when it
// detects a DequantizeLinear -> Pooling -> QuantizeLinear pattern.
// Uses optimised SIMD kernels; OPENCV (CPU) backend only.
class CV_EXPORTS Pool2Int8Layer : public Layer
{
public:
static Ptr<Pool2Int8Layer> create(const LayerParams& params);
int input_zp, output_zp;
float input_sc, output_sc;
std::vector<int> kernel_shape, strides, dilations, pads;
AutoPadding auto_pad;
bool ceil_mode;
bool is_global_pooling;
bool is_max_pool;
};
class CV_EXPORTS AveragePoolLayer : public Layer
{
public:
@@ -559,6 +596,7 @@ CV__DNN_INLINE_NS_BEGIN
public:
int input_zp, output_zp;
float input_sc, output_sc;
int output_type; // CV_8S or CV_8U
// quantization type flag. The perChannel default is true, that means it contains the parameters
// of per-Channel quantization. Otherwise, that means this layer contains per-Tensor quantized parameters.
@@ -1214,6 +1252,17 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<EltwiseLayerInt8> create(const LayerParams &params);
};
class CV_EXPORTS Eltwise2Int8Layer : public Layer
{
public:
static Ptr<Eltwise2Int8Layer> create(const LayerParams& params);
std::vector<float> scales;
std::vector<int> zeropoints;
float output_sc;
int output_zp;
};
class CV_EXPORTS NaryEltwiseLayer : public Layer
{
public:
modules/dnn/src/graph_fusion_qdq.cpp
+531 -369
View File
@@ -51,7 +51,31 @@ struct ModelFusionQDQ
Ptr<Layer> createFusedLayer(const LayerParams& src) const
{
LayerParams params = src;
return LayerFactory::createLayerInstance(params.type, params);
Ptr<Layer> layer = LayerFactory::createLayerInstance(params.type, params);
if (!layer.empty())
layer->netimpl = netimpl;
return layer;
}
static bool getInt8OutputParams(Layer* layer, float& sc, int& zp)
{
if (auto* p = dynamic_cast<Conv2Int8Layer*>(layer))
{ sc = p->output_sc; zp = p->output_zp; return true; }
if (auto* p = dynamic_cast<Eltwise2Int8Layer*>(layer))
{ sc = p->output_sc; zp = p->output_zp; return true; }
if (auto* p = dynamic_cast<InnerProductLayerInt8*>(layer))
{ sc = p->output_sc; zp = p->output_zp; return true; }
return false;
}
static void setInt8OutputParams(Layer* layer, float sc, int zp)
{
if (auto* p = dynamic_cast<Conv2Int8Layer*>(layer))
{ p->output_sc = sc; p->output_zp = zp; }
else if (auto* p = dynamic_cast<Eltwise2Int8Layer*>(layer))
{ p->output_sc = sc; p->output_zp = zp; }
else if (auto* p = dynamic_cast<InnerProductLayerInt8*>(layer))
{ p->output_sc = sc; p->output_zp = zp; }
}
template<typename LayerT>
@@ -86,8 +110,12 @@ struct ModelFusionQDQ
std::vector<int> producer_of(nargs, -1);
std::vector<Ptr<Layer> > newprog;
std::vector<Arg> fused_inputs;
std::set<int> skip_indices;
std::vector<Arg> override_outputs;
std::map<int, std::pair<Arg, Eltwise2Int8Layer*>> relu_to_eltwise;
for (i = 0; i < nops; i++) {
if (skip_indices.count((int)i)) continue;
const Ptr<Layer>& layer = prog[i];
Layer* layer_ptr = (Layer*)layer.get();
int fused_layer_idx = -1;
@@ -102,7 +130,8 @@ struct ModelFusionQDQ
const std::vector<Arg>& outputs = layer->outputs;
size_t ninputs = inputs.size();
removed_args.clear();
fused_inputs.clear(); // leave it empty to re-use original fused node inputs as-is.
fused_inputs.clear();
override_outputs.clear();
for(;;) {
Arg q_data_in, out_scale, out_zp;
@@ -128,13 +157,12 @@ struct ModelFusionQDQ
}
dq_ptrs.push_back(dq);
dq_prog_indices.push_back(dq_idx);
int8_inputs.push_back(dq->inputs[0]); // int8 quantized tensor
int8_inputs.push_back(dq->inputs[0]);
}
const int eltwise_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
const bool eltwise_out_int8 = (eltwise_out_type == CV_8S || eltwise_out_type == CV_8U);
if (dq_ptrs.size() == add->inputs.size() && eltwise_out_int8) {
// Read per-input scale and zero-point from the DQ const args.
vector<float> in_scales(2);
vector<int> in_zps(2);
bool eltwise_in_int8 = true;
@@ -156,16 +184,15 @@ struct ModelFusionQDQ
? (int)elt_out_zp_m.at<uint8_t>(0)
: (int)elt_out_zp_m.at<int8_t>(0);
LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "EltwiseInt8");
LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "Eltwise2Int8");
eltwiseParams.blobs.clear();
eltwiseParams.set("input_scales", DictValue::arrayReal(in_scales.data(), (int)in_scales.size()));
eltwiseParams.set("input_zeropoints", DictValue::arrayInt(in_zps.data(), (int)in_zps.size()));
eltwiseParams.set("scales", out_scale_val);
eltwiseParams.set("zeropoints", out_zp_val);
Ptr<Layer> eltwiseInt8 = createFusedLayer(eltwiseParams);
if (!eltwiseInt8.empty()) {
auto* elt = dynamic_cast<EltwiseLayerInt8*>(eltwiseInt8.get());
CV_Assert(elt);
elt->scales = in_scales;
elt->zeropoints = in_zps;
elt->output_sc = out_scale_val;
elt->output_zp = out_zp_val;
CV_Assert(dynamic_cast<Eltwise2Int8Layer*>(eltwiseInt8.get()));
fused_layer_idx = add_layer_idx;
newprog[add_layer_idx] = eltwiseInt8;
fused_inputs.swap(int8_inputs);
@@ -185,114 +212,124 @@ struct ModelFusionQDQ
ReLULayer* relu = 0;
if (getQdqPatternContext<ReLULayer>(layer_ptr, ninputs, inputs, producer_of,
newprog, q_data_in, out_scale, out_zp,
relu_layer_idx, relu) &&
relu->inputs.size() == 1) {
Arg relu_in = relu->inputs[0];
int dq_idx = producer_of.at(relu_in.idx);
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
relu_layer_idx, relu) && relu->inputs.size() == 1) {
Arg relu_in = relu->inputs[0];
int dq_idx = producer_of.at(relu_in.idx);
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
const int relu_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
const bool relu_out_int8 = (relu_out_type == CV_8S || relu_out_type == CV_8U);
const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U);
if (dq && dq->inputs.size() >= 3 &&
relu_in_int8 && relu_out_int8 &&
usecounts.at(relu_in.idx) == 1) {
const float inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
const Mat& relu_zp_m = netimpl->argTensor(dq->inputs[2]);
const int inp_zp = relu_zp_m.depth() == CV_8U
? (int)relu_zp_m.at<uint8_t>(0)
: (int)relu_zp_m.at<int8_t>(0);
const float out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& out_zp_relu_m = netimpl->argTensor(out_zp);
const int out_zp_i = out_zp_relu_m.depth() == CV_8U
? (int)out_zp_relu_m.at<uint8_t>(0)
: (int)out_zp_relu_m.at<int8_t>(0);
const int relu_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
const bool relu_out_int8 = (relu_out_type == CV_8S || relu_out_type == CV_8U);
const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U);
if (dq && dq->inputs.size() >= 3 &&
relu_in_int8 && relu_out_int8 &&
usecounts.at(relu_in.idx) == 1) {
const float inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
const Mat& relu_zp_m = netimpl->argTensor(dq->inputs[2]);
const int inp_zp = relu_zp_m.depth() == CV_8U
? (int)relu_zp_m.at<uint8_t>(0)
: (int)relu_zp_m.at<int8_t>(0);
const float out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& out_zp_relu_m = netimpl->argTensor(out_zp);
const int out_zp_i = out_zp_relu_m.depth() == CV_8U
? (int)out_zp_relu_m.at<uint8_t>(0)
: (int)out_zp_relu_m.at<int8_t>(0);
if (inp_sc > 0.f && out_sc > 0.f) {
const bool isU8 = (relu_in_type == CV_8U);
Mat lookUpTable(1, 256, isU8 ? CV_8U : CV_8S);
if (isU8) {
uint8_t* table = lookUpTable.ptr<uint8_t>();
for (int t = 0; t < 256; t++) {
float x = inp_sc * (t - inp_zp);
float y = std::max(0.0f, x);
int quantized = out_zp_i + cvRound(y / out_sc);
table[t] = saturate_cast<uint8_t>(quantized);
}
} else {
int8_t* table = lookUpTable.ptr<int8_t>();
for (int t = -128; t < 128; t++) {
float x = inp_sc * (t - inp_zp);
float y = std::max(0.0f, x);
int quantized = out_zp_i + cvRound(y / out_sc);
table[t + 128] = saturate_cast<int8_t>(quantized);
}
if (inp_sc > 0.f && out_sc > 0.f) {
const bool isU8 = (relu_in_type == CV_8U);
Mat lookUpTable(1, 256, isU8 ? CV_8U : CV_8S);
if (isU8) {
uint8_t* table = lookUpTable.ptr<uint8_t>();
for (int t = 0; t < 256; t++) {
float x = inp_sc * (t - inp_zp);
float y = std::max(0.0f, x);
int quantized = out_zp_i + cvRound(y / out_sc);
table[t] = saturate_cast<uint8_t>(quantized);
}
LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8");
reluInt8Params.blobs.clear();
Ptr<Layer> reluInt8 = createFusedLayer(reluInt8Params);
if (!reluInt8.empty()) {
auto* reluInt8Layer = dynamic_cast<ActivationLayerInt8*>(reluInt8.get());
CV_Assert(reluInt8Layer);
reluInt8Layer->input_sc = inp_sc;
reluInt8Layer->input_zp = inp_zp;
reluInt8Layer->output_sc = out_sc;
reluInt8Layer->output_zp = out_zp_i;
reluInt8Layer->activationLUT = lookUpTable;
fused_layer_idx = relu_layer_idx;
newprog[relu_layer_idx] = reluInt8;
fused_inputs.assign(1, dq->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(relu_in);
newprog[dq_idx] = Ptr<Layer>();
break;
} else {
int8_t* table = lookUpTable.ptr<int8_t>();
for (int t = -128; t < 128; t++) {
float x = inp_sc * (t - inp_zp);
float y = std::max(0.0f, x);
int quantized = out_zp_i + cvRound(y / out_sc);
table[t + 128] = saturate_cast<int8_t>(quantized);
}
}
LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8");
reluInt8Params.blobs.clear();
Ptr<Layer> reluInt8 = createFusedLayer(reluInt8Params);
if (!reluInt8.empty()) {
auto* reluInt8Layer = dynamic_cast<ActivationLayerInt8*>(reluInt8.get());
CV_Assert(reluInt8Layer);
reluInt8Layer->input_sc = inp_sc;
reluInt8Layer->input_zp = inp_zp;
reluInt8Layer->output_sc = out_sc;
reluInt8Layer->output_zp = out_zp_i;
reluInt8Layer->activationLUT = lookUpTable;
fused_layer_idx = relu_layer_idx;
newprog[relu_layer_idx] = reluInt8;
fused_inputs.assign(1, dq->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(relu_in);
newprog[dq_idx] = Ptr<Layer>();
break;
}
}
}
}
// Compound pattern: DQ, DQ -> Add -> ReLU -> QuantizeLinear
// Common in ResNet residual blocks. Fuses into EltwiseInt8 with activation LUT.
{
int relu_layer_idx2 = -1;
ReLULayer* relu2 = 0;
Arg q_data_in2, out_scale2, out_zp2;
if (getQdqPatternContext<ReLULayer>(layer_ptr, ninputs, inputs, producer_of,
newprog, q_data_in2, out_scale2, out_zp2,
relu_layer_idx2, relu2) &&
relu2->inputs.size() == 1) {
QuantizeLinearLayer* ql_compound = dynamic_cast<QuantizeLinearLayer*>(layer_ptr);
if (ql_compound && ninputs == 3) {
Arg q_inp = inputs[0];
int relu_layer_idx2 = (q_inp.idx < (int)producer_of.size()) ? producer_of.at(q_inp.idx) : -1;
ReLULayer* relu2 = getLayer<ReLULayer>(newprog, relu_layer_idx2);
(void)relu_layer_idx2;
if (relu2 && relu2->inputs.size() == 1) {
Arg relu_in2 = relu2->inputs[0];
int add_idx2 = producer_of.at(relu_in2.idx);
NaryEltwiseLayer* add2 = getLayer<NaryEltwiseLayer>(newprog, add_idx2);
int relu_in_uc = usecounts.at(relu_in2.idx);
if (add2 && add2->inputs.size() >= 2 &&
usecounts.at(relu_in2.idx) == 1) {
relu_in_uc == 1) {
vector<DequantizeLinearLayer*> dq_ptrs2;
vector<int> dq_prog_indices2;
vector<Arg> int8_inputs2;
for (size_t k = 0; k < add2->inputs.size(); k++) {
const Arg& add_inp = add2->inputs[k];
int dq_idx2 = producer_of.at(add_inp.idx);
DequantizeLinearLayer* dq2 =
getLayer<DequantizeLinearLayer>(newprog, dq_idx2);
if (dq2 && dq2->inputs.size() >= 3 &&
usecounts.at(add_inp.idx) == 1) {
DequantizeLinearLayer* dq2 = getLayer<DequantizeLinearLayer>(newprog, dq_idx2);
if (dq2 && dq2->inputs.size() >= 3) {
dq_ptrs2.push_back(dq2);
dq_prog_indices2.push_back(dq_idx2);
int8_inputs2.push_back(dq2->inputs[0]);
} else {
int arg_type = netimpl->argData(add_inp).type;
if (arg_type == CV_8S || arg_type == CV_8U) {
bool found_int8 = (arg_type == CV_8S || arg_type == CV_8U);
Arg int8_arg = add_inp;
if (!found_int8) {
auto it = relu_to_eltwise.find(add_inp.idx);
if (it != relu_to_eltwise.end()) {
int8_arg = it->second.first;
found_int8 = true;
}
}
if (found_int8) {
dq_ptrs2.push_back(nullptr);
dq_prog_indices2.push_back(-1);
int8_inputs2.push_back(add_inp);
int8_inputs2.push_back(int8_arg);
} else {
break; // not int8, can't fuse
break;
}
}
}
const int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
if (elt_out_type2 < 0) {
const Mat& zp_tensor = netimpl->argTensor(inputs[2]);
elt_out_type2 = !zp_tensor.empty() ? zp_tensor.type() : CV_8S;
}
const bool elt_out_int82 = (elt_out_type2 == CV_8S || elt_out_type2 == CV_8U);
if (int8_inputs2.size() == add2->inputs.size() && elt_out_int82) {
vector<float> in_scales2(2);
@@ -301,6 +338,10 @@ struct ModelFusionQDQ
for (int k = 0; k < 2; k++) {
if (dq_ptrs2[k]) {
int it = netimpl->argData(dq_ptrs2[k]->inputs[0]).type;
if (it < 0) {
const Mat& zp_infer = netimpl->argTensor(dq_ptrs2[k]->inputs[2]);
it = !zp_infer.empty() ? zp_infer.type() : CV_8S;
}
elt_in_int82 = elt_in_int82 && (it == CV_8S || it == CV_8U);
in_scales2[k] = netimpl->argTensor(dq_ptrs2[k]->inputs[1]).at<float>(0);
const Mat& zp_m2 = netimpl->argTensor(dq_ptrs2[k]->inputs[2]);
@@ -308,80 +349,62 @@ struct ModelFusionQDQ
? (int)zp_m2.at<uint8_t>(0)
: (int)zp_m2.at<int8_t>(0);
} else {
auto relu_it = relu_to_eltwise.find(add2->inputs[k].idx);
if (relu_it != relu_to_eltwise.end()) {
Eltwise2Int8Layer* elt2 = relu_it->second.second;
if (elt2) {
in_scales2[k] = elt2->output_sc;
in_zps2[k] = elt2->output_zp;
continue;
}
}
int prod_idx = producer_of.at(add2->inputs[k].idx);
Layer* prod = prod_idx >= 0 && !newprog[prod_idx].empty()
? newprog[prod_idx].get() : nullptr;
ConvolutionLayerInt8* ci = prod ? dynamic_cast<ConvolutionLayerInt8*>(prod) : nullptr;
EltwiseLayerInt8* ei = prod ? dynamic_cast<EltwiseLayerInt8*>(prod) : nullptr;
InnerProductLayerInt8* fi = prod ? dynamic_cast<InnerProductLayerInt8*>(prod) : nullptr;
if (ci) { in_scales2[k] = ci->output_sc; in_zps2[k] = ci->output_zp; }
else if (ei) { in_scales2[k] = ei->output_sc; in_zps2[k] = ei->output_zp; }
else if (fi) { in_scales2[k] = fi->output_sc; in_zps2[k] = fi->output_zp; }
else { elt_in_int82 = false; }
if (!prod || !getInt8OutputParams(prod, in_scales2[k], in_zps2[k]))
elt_in_int82 = false;
}
}
if (elt_in_int82) {
float out_sc2 = netimpl->argTensor(out_scale2).at<float>(0);
const Mat& out_zp_m2 = netimpl->argTensor(out_zp2);
float out_sc2 = netimpl->argTensor(inputs[1]).at<float>(0);
const Mat& out_zp_m2 = netimpl->argTensor(inputs[2]);
int out_zp_val2 = out_zp_m2.depth() == CV_8U
? (int)out_zp_m2.at<uint8_t>(0)
: (int)out_zp_m2.at<int8_t>(0);
if (out_sc2 > 0.f) {
LayerParams eltParams = makeLayerParamsFromOriginal(add2, "EltwiseInt8");
int relu_out_uc = usecounts.at(q_inp.idx);
LayerParams eltParams = makeLayerParamsFromOriginal(add2, "Eltwise2Int8");
eltParams.blobs.clear();
eltParams.set("input_scales", DictValue::arrayReal(in_scales2.data(), (int)in_scales2.size()));
eltParams.set("input_zeropoints", DictValue::arrayInt(in_zps2.data(), (int)in_zps2.size()));
eltParams.set("scales", out_sc2);
eltParams.set("zeropoints", out_zp_val2);
eltParams.set("with_relu", true);
Ptr<Layer> eltInt8 = createFusedLayer(eltParams);
if (!eltInt8.empty()) {
auto* elt = dynamic_cast<EltwiseLayerInt8*>(eltInt8.get());
CV_Assert(elt);
elt->scales = in_scales2;
elt->zeropoints = in_zps2;
elt->output_sc = out_sc2;
elt->output_zp = out_zp_val2;
CV_Assert(dynamic_cast<Eltwise2Int8Layer*>(eltInt8.get()));
const bool isU8 = (elt_out_type2 == CV_8U);
Mat lut(1, 256, isU8 ? CV_8U : CV_8S);
if (isU8) {
uint8_t* tbl = lut.ptr<uint8_t>();
for (int t = 0; t < 256; t++) {
float x = out_sc2 * (t - out_zp_val2);
float y = std::max(0.0f, x);
tbl[t] = saturate_cast<uint8_t>(out_zp_val2 + cvRound(y / out_sc2));
fused_inputs = int8_inputs2;
if (relu_out_uc <= 1) {
fused_layer_idx = add_idx2;
newprog[add_idx2] = eltInt8;
newprog[relu_layer_idx2] = Ptr<Layer>();
removed_args.push_back(q_inp); // relu_out
removed_args.push_back(relu_in2); // add_out
for (size_t dk = 0; dk < add2->inputs.size(); dk++) {
removed_args.push_back(add2->inputs[dk]);
}
for (int dq_prog_idx : dq_prog_indices2) {
if (dq_prog_idx >= 0)
newprog[dq_prog_idx] = Ptr<Layer>();
}
} else {
int8_t* tbl = lut.ptr<int8_t>();
for (int t = -128; t < 128; t++) {
float x = out_sc2 * (t - out_zp_val2);
float y = std::max(0.0f, x);
tbl[t + 128] = saturate_cast<int8_t>(out_zp_val2 + cvRound(y / out_sc2));
}
}
LayerParams reluActParams;
reluActParams.name = relu2->name;
reluActParams.type = "ReLUInt8";
Ptr<Layer> reluAct = createFusedLayer(reluActParams);
if (!reluAct.empty()) {
auto* reluActLayer = dynamic_cast<ActivationLayerInt8*>(reluAct.get());
if (reluActLayer) {
reluActLayer->input_sc = out_sc2;
reluActLayer->input_zp = out_zp_val2;
reluActLayer->output_sc = out_sc2;
reluActLayer->output_zp = out_zp_val2;
reluActLayer->activationLUT = lut;
eltInt8->setActivation(reluAct.dynamicCast<ActivationLayer>());
}
}
fused_layer_idx = add_idx2;
newprog[add_idx2] = eltInt8;
newprog[relu_layer_idx2] = Ptr<Layer>();
fused_inputs.swap(int8_inputs2);
removed_args.push_back(q_data_in2);
removed_args.push_back(relu_in2);
for (const Arg& add_inp : add2->inputs)
removed_args.push_back(add_inp);
for (int dq_prog_idx : dq_prog_indices2) {
if (dq_prog_idx >= 0)
newprog[dq_prog_idx] = Ptr<Layer>();
int new_idx = (int)newprog.size();
newprog.push_back(eltInt8);
fused_layer_idx = new_idx;
usecounts.at(q_inp.idx) -= 1;
Eltwise2Int8Layer* elt2ptr = dynamic_cast<Eltwise2Int8Layer*>(eltInt8.get());
relu_to_eltwise[q_inp.idx] = {outputs[0], elt2ptr};
}
break;
}
@@ -406,8 +429,6 @@ struct ModelFusionQDQ
DequantizeLinearLayer* dq_x = getLayer<DequantizeLinearLayer>(newprog, dq_x_idx);
DequantizeLinearLayer* dq_w = getLayer<DequantizeLinearLayer>(newprog, dq_w_idx);
// Allow usecounts > 1 for conv input (shared DQ output at stage transitions)
// The int8 data (DQ's input[0]) can be shared safely.
if (dq_x && dq_w &&
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
usecounts.at(conv_w.idx) == 1) {
@@ -433,133 +454,129 @@ struct ModelFusionQDQ
Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]);
const Mat& x_zp_m = netimpl->argTensor(dq_x->inputs[2]);
if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims >= 3) {
const int outCn = w_q.size[0];
Mat wt_sc = (w_sc_m.total() == (size_t)outCn)
? w_sc_m.reshape(1, 1)
: Mat(1, outCn, CV_32F, Scalar(w_sc_m.at<float>(0))).clone();
bool per_channel = w_sc_m.total() == (size_t)outCn;
const int outCn = w_q.size[0];
Mat wt_sc = (w_sc_m.total() == (size_t)outCn)
? w_sc_m.reshape(1, 1)
: Mat(1, outCn, CV_32F, Scalar(w_sc_m.at<float>(0))).clone();
bool per_channel = w_sc_m.total() == (size_t)outCn;
bool all_wzp_zero = true;
if (w_zp_m.total() > 1 && w_zp_m.total() != (size_t)outCn)
all_wzp_zero = false;
for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) {
int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at<int8_t>((int)t)
: (int)w_zp_m.at<uint8_t>((int)t);
if (wz != 0) all_wzp_zero = false;
}
bool all_wzp_zero = true;
if (w_zp_m.total() > 1 && w_zp_m.total() != (size_t)outCn)
all_wzp_zero = false;
for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) {
int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at<int8_t>((int)t)
: (int)w_zp_m.at<uint8_t>((int)t);
if (wz != 0) all_wzp_zero = false;
}
bool symmetric_pads = true;
size_t npads = conv->pads.size();
size_t ndims_pad = npads / 2;
for (size_t d = 0; d < ndims_pad && symmetric_pads; d++) {
if (conv->pads[d] != conv->pads[d + ndims_pad])
symmetric_pads = false;
}
bool symmetric_pads = true;
size_t npads = conv->pads.size();
size_t ndims_pad = npads / 2;
for (size_t d = 0; d < ndims_pad && symmetric_pads; d++) {
if (conv->pads[d] != conv->pads[d + ndims_pad])
symmetric_pads = false;
}
if (all_wzp_zero && symmetric_pads) {
Mat bias = Mat::zeros(1, outCn, CV_32S);
bool biasOk = true;
int dq_bias_idx = -1;
if (conv->inputs.size() == 3) {
if (netimpl->isConstArg(conv->inputs[2])) {
Mat b = netimpl->argTensor(conv->inputs[2]);
if (b.empty() || b.total() != (size_t)outCn) {
biasOk = false;
} else if (b.depth() == CV_32S) {
bias = b.reshape(1, 1);
} else if (b.depth() == CV_32F || b.depth() == CV_64F) {
Mat b1 = b.reshape(1, 1);
for (int oc = 0; oc < outCn; oc++) {
const float b_real = b1.depth() == CV_32F
? b1.at<float>(oc)
: (float)b1.at<double>(oc);
const float denom = inp_sc * wt_sc.at<float>(oc);
if (std::abs(denom) < 1e-12f) { biasOk = false; break; }
bias.at<int>(oc) = cvRound(b_real / denom);
}
} else {
biasOk = false;
if (all_wzp_zero && symmetric_pads) {
Mat bias = Mat::zeros(1, outCn, CV_32S);
bool biasOk = true;
int dq_bias_idx = -1;
if (conv->inputs.size() == 3) {
if (netimpl->isConstArg(conv->inputs[2])) {
Mat b = netimpl->argTensor(conv->inputs[2]);
if (b.empty() || b.total() != (size_t)outCn) {
biasOk = false;
} else if (b.depth() == CV_32S) {
bias = b.reshape(1, 1);
} else if (b.depth() == CV_32F || b.depth() == CV_64F) {
Mat b1 = b.reshape(1, 1);
for (int oc = 0; oc < outCn; oc++) {
const float b_real = b1.depth() == CV_32F
? b1.at<float>(oc)
: (float)b1.at<double>(oc);
const float denom = inp_sc * wt_sc.at<float>(oc);
if (std::abs(denom) < 1e-12f) { biasOk = false; break; }
bias.at<int>(oc) = cvRound(b_real / denom);
}
} else {
dq_bias_idx = producer_of.at(conv->inputs[2].idx);
DequantizeLinearLayer* dq_b =
getLayer<DequantizeLinearLayer>(newprog, dq_bias_idx);
if (!dq_b || dq_b->inputs.size() < 2 ||
usecounts.at(conv->inputs[2].idx) != 1 ||
!netimpl->isConstArg(dq_b->inputs[0])) {
biasOk = false;
}
} else {
dq_bias_idx = producer_of.at(conv->inputs[2].idx);
DequantizeLinearLayer* dq_b =
getLayer<DequantizeLinearLayer>(newprog, dq_bias_idx);
if (!dq_b || dq_b->inputs.size() < 2 ||
usecounts.at(conv->inputs[2].idx) != 1 ||
!netimpl->isConstArg(dq_b->inputs[0])) {
biasOk = false;
} else {
Mat bq = netimpl->argTensor(dq_b->inputs[0]);
if (bq.empty() || bq.total() != (size_t)outCn || bq.depth() != CV_32S)
biasOk = false;
} else {
Mat bq = netimpl->argTensor(dq_b->inputs[0]);
if (bq.empty() || bq.total() != (size_t)outCn || bq.depth() != CV_32S)
biasOk = false;
else
bias = bq.reshape(1, 1);
}
else
bias = bq.reshape(1, 1);
}
}
if (!biasOk)
break;
}
if (!biasOk)
break;
const bool inputIsU8 =
(netimpl->argData(dq_x->inputs[0]).type == CV_8U) ||
(x_zp_m.depth() == CV_8U);
const int inp_zp_kernel = inputIsU8 ? (inp_zp - 128) : inp_zp;
Mat weights_2d = w_q.reshape(1, outCn);
Mat biasFused(1, outCn, CV_32S);
Mat outputMultiplier(1, outCn, CV_32F);
for (int oc = 0; oc < outCn; oc++) {
biasFused.at<int>(oc) = bias.at<int>(oc) - inp_zp_kernel * (int)cv::sum(weights_2d.row(oc))[0];
outputMultiplier.at<float>(oc) = (inp_sc * wt_sc.at<float>(oc)) / out_sc;
}
const bool inputIsU8 =
(netimpl->argData(dq_x->inputs[0]).type == CV_8U) ||
(x_zp_m.depth() == CV_8U);
const int inp_zp_kernel = inputIsU8 ? (inp_zp - 128) : inp_zp;
Mat weights_2d = w_q.reshape(1, outCn);
Mat biasFused(1, outCn, CV_32S);
Mat outputMultiplier(1, outCn, CV_32F);
for (int oc = 0; oc < outCn; oc++) {
biasFused.at<int>(oc) = bias.at<int>(oc) - inp_zp_kernel * (int)cv::sum(weights_2d.row(oc))[0];
outputMultiplier.at<float>(oc) = (inp_sc * wt_sc.at<float>(oc)) / out_sc;
}
LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "ConvolutionInt8");
{
int kndims = w_q.dims - 2;
std::vector<int> ksize(kndims);
for (int d = 0; d < kndims; d++)
ksize[d] = w_q.size[d + 2];
convInt8Params.set("kernel_size", DictValue::arrayInt(ksize.data(), kndims));
if (!conv->strides.empty())
convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size()));
if (!conv->dilations.empty())
convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size()));
if (!conv->pads.empty())
convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size()));
LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "Conv2Int8");
{
if (!conv->strides.empty())
convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size()));
if (!conv->dilations.empty())
convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size()));
if (!conv->pads.empty())
convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size()));
}
convInt8Params.set("num_output", outCn);
convInt8Params.set("group", conv->ngroups);
convInt8Params.set("input_scale", inp_sc);
convInt8Params.set("input_zeropoint", inp_zp);
convInt8Params.set("scales", out_sc);
convInt8Params.set("zeropoints", out_zp);
convInt8Params.set("per_channel", per_channel);
convInt8Params.set("input_is_u8", inputIsU8);
convInt8Params.blobs.resize(3);
convInt8Params.blobs[0] = w_q;
convInt8Params.blobs[1] = biasFused;
convInt8Params.blobs[2] = outputMultiplier;
Ptr<Layer> convInt8 = createFusedLayer(convInt8Params);
if (!convInt8.empty()) {
auto* convInt8Layer = dynamic_cast<Conv2Int8Layer*>(convInt8.get());
CV_Assert(convInt8Layer);
fused_layer_idx = conv_layer_idx;
newprog[conv_layer_idx] = convInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(conv_w);
if (conv->inputs.size() == 3) {
removed_args.push_back(conv->inputs[2]);
if (dq_bias_idx >= 0)
newprog[dq_bias_idx] = Ptr<Layer>();
}
convInt8Params.set("num_output", outCn);
convInt8Params.set("group", conv->ngroups);
convInt8Params.blobs.resize(3);
convInt8Params.blobs[0] = w_q;
convInt8Params.blobs[1] = biasFused;
convInt8Params.blobs[2] = outputMultiplier;
Ptr<Layer> convInt8 = createFusedLayer(convInt8Params);
if (!convInt8.empty()) {
auto* convInt8Layer = dynamic_cast<ConvolutionLayerInt8*>(convInt8.get());
CV_Assert(convInt8Layer);
convInt8Layer->input_zp = inp_zp;
convInt8Layer->input_sc = inp_sc;
convInt8Layer->output_zp = out_zp;
convInt8Layer->output_sc = out_sc;
convInt8Layer->per_channel = per_channel;
fused_layer_idx = conv_layer_idx;
newprog[conv_layer_idx] = convInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(conv_w);
if (conv->inputs.size() == 3) {
removed_args.push_back(conv->inputs[2]);
if (dq_bias_idx >= 0)
newprog[dq_bias_idx] = Ptr<Layer>();
}
if (usecounts.at(conv_x.idx) == 1) {
removed_args.push_back(conv_x);
newprog[dq_x_idx] = Ptr<Layer>();
}
newprog[dq_w_idx] = Ptr<Layer>();
break;
if (usecounts.at(conv_x.idx) == 1) {
removed_args.push_back(conv_x);
newprog[dq_x_idx] = Ptr<Layer>();
}
newprog[dq_w_idx] = Ptr<Layer>();
break;
}
}
}
}
}
@@ -577,9 +594,17 @@ struct ModelFusionQDQ
DequantizeLinearLayer* dq_w = getLayer<DequantizeLinearLayer>(newprog, dq_w_idx);
float inp_sc = 0.f, out_sc = 0.f;
int inp_zp = 0, out_zp_i = 0;
const int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
if (fc_out_type < 0) {
const Mat& zp_t = netimpl->argTensor(inputs[2]);
fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U);
const int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1;
int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1;
if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) {
const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]);
fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U);
if (dq_x && dq_w &&
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
@@ -587,8 +612,9 @@ struct ModelFusionQDQ
usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) {
inp_sc = netimpl->argTensor(dq_x->inputs[1]).at<float>(0);
const Mat& fc_zp_m = netimpl->argTensor(dq_x->inputs[2]);
inp_zp = fc_zp_m.depth() == CV_8U
? (int)fc_zp_m.at<uint8_t>(0)
? (int)fc_zp_m.at<uint8_t>(0) - 128
: (int)fc_zp_m.at<int8_t>(0);
out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& fc_out_zp_m = netimpl->argTensor(out_zp);
@@ -638,6 +664,7 @@ struct ModelFusionQDQ
fcInt8Layer->input_sc = inp_sc;
fcInt8Layer->output_zp = out_zp_i;
fcInt8Layer->output_sc = out_sc;
fcInt8Layer->output_type = fc_out_type;
fcInt8Layer->per_channel = per_channel;
fused_layer_idx = mm_layer_idx;
newprog[mm_layer_idx] = fcInt8;
@@ -654,112 +681,211 @@ struct ModelFusionQDQ
}
}
int pool_layer_idx = -1;
PoolingLayer* pool = 0;
if (getQdqPatternContext<PoolingLayer>(layer_ptr, ninputs, inputs, producer_of,
newprog, q_data_in, out_scale, out_zp,
pool_layer_idx, pool) &&
pool->inputs.size() == 1) {
Arg pool_in = pool->inputs[0];
int dq_idx = producer_of.at(pool_in.idx);
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
float inp_sc = 0.f, out_sc = 0.f;
int inp_zp = 0, out_zp_i = 0;
const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U);
const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U);
if (dq && dq->inputs.size() >= 3 &&
pool_in_int8 && pool_out_int8 &&
usecounts.at(pool_in.idx) == 1) {
inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]);
inp_zp = pool_zp_m.depth() == CV_8U
? (int)pool_zp_m.at<uint8_t>(0)
: (int)pool_zp_m.at<int8_t>(0);
out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& pool_out_zp_m = netimpl->argTensor(out_zp);
out_zp_i = pool_out_zp_m.depth() == CV_8U
? (int)pool_out_zp_m.at<uint8_t>(0)
: (int)pool_out_zp_m.at<int8_t>(0);
bool isGlobalAve = pool->globalPooling;
bool isMax = !isGlobalAve;
if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) ||
(isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) {
LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "PoolingInt8");
poolInt8Params.blobs.clear();
Ptr<Layer> poolInt8 = createFusedLayer(poolInt8Params);
if (!poolInt8.empty()) {
auto* poolInt8Layer = dynamic_cast<PoolingLayerInt8*>(poolInt8.get());
CV_Assert(poolInt8Layer);
const String poolInt8Type = static_cast<Layer&>(*poolInt8Layer).type;
std::vector<Mat> poolInt8Blobs = poolInt8Layer->blobs;
static_cast<PoolingLayer&>(*poolInt8Layer) = *pool;
static_cast<Layer&>(*poolInt8Layer).type = poolInt8Type;
poolInt8Layer->blobs = poolInt8Blobs;
poolInt8Layer->input_sc = inp_sc;
poolInt8Layer->input_zp = inp_zp;
poolInt8Layer->output_sc = out_sc;
poolInt8Layer->output_zp = out_zp_i;
fused_layer_idx = pool_layer_idx;
newprog[pool_layer_idx] = poolInt8;
fused_inputs.assign(1, dq->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(pool_in);
newprog[dq_idx] = Ptr<Layer>();
break;
int add_bias_idx = -1;
NaryEltwiseLayer* add_bias = 0;
if (getQdqPatternContext<NaryEltwiseLayer>(layer_ptr, ninputs, inputs, producer_of,
newprog, q_data_in, out_scale, out_zp,
add_bias_idx, add_bias) &&
add_bias && add_bias->inputs.size() == 2) {
int mm_inp_k = -1;
int bias_inp_k = -1;
MatMulLayer* mm2 = nullptr;
int mm2_idx = -1;
for (int k = 0; k < 2; k++) {
int prod_idx = producer_of.at(add_bias->inputs[k].idx);
MatMulLayer* candidate = getLayer<MatMulLayer>(newprog, prod_idx);
if (candidate && candidate->inputs.size() == 2) {
mm_inp_k = k;
mm2 = candidate;
mm2_idx = prod_idx;
} else if (netimpl->isConstArg(add_bias->inputs[k])) {
bias_inp_k = k;
}
}
if (mm2 && bias_inp_k >= 0 && mm_inp_k >= 0 &&
usecounts.at(add_bias->inputs[mm_inp_k].idx) == 1) {
const Arg mm_x = mm2->inputs[0];
const Arg mm_w = mm2->inputs[1];
int dq_x_idx = producer_of.at(mm_x.idx);
int dq_w_idx = producer_of.at(mm_w.idx);
DequantizeLinearLayer* dq_x = getLayer<DequantizeLinearLayer>(newprog, dq_x_idx);
DequantizeLinearLayer* dq_w = getLayer<DequantizeLinearLayer>(newprog, dq_w_idx);
int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
if (fc_out_type < 0) {
const Mat& zp_t = netimpl->argTensor(inputs[2]);
fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U);
int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1;
if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) {
const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]);
fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U);
if (dq_x && dq_w &&
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
fc_in_int8 && fc_out_int8 &&
usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) {
float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at<float>(0);
const Mat& fc_zp_m = netimpl->argTensor(dq_x->inputs[2]);
int inp_zp = fc_zp_m.depth() == CV_8U
? (int)fc_zp_m.at<uint8_t>(0) - 128
: (int)fc_zp_m.at<int8_t>(0);
float out_sc_val = netimpl->argTensor(out_scale).at<float>(0);
const Mat& fc_out_zp_m = netimpl->argTensor(out_zp);
int out_zp_i = fc_out_zp_m.depth() == CV_8U
? (int)fc_out_zp_m.at<uint8_t>(0)
: (int)fc_out_zp_m.at<int8_t>(0);
if (inp_sc > 0.f && out_sc_val > 0.f) {
Mat w_q = netimpl->argTensor(dq_w->inputs[0]);
Mat w_sc_m = netimpl->argTensor(dq_w->inputs[1]);
Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]);
if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims == 2) {
bool all_wzp_zero = true;
for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) {
int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at<int8_t>((int)t)
: (int)w_zp_m.at<uint8_t>((int)t);
if (wz != 0) all_wzp_zero = false;
}
if (all_wzp_zero) {
Mat weights = w_q.t();
int outCn = weights.size[0];
Mat wt_sc = (w_sc_m.total() == (size_t)outCn)
? w_sc_m.reshape(1, 1)
: Mat(1, outCn, CV_32F, Scalar(w_sc_m.at<float>(0))).clone();
bool per_channel = w_sc_m.total() == (size_t)outCn;
// Fuse the float bias into int32 bias
Mat float_bias = netimpl->argTensor(add_bias->inputs[bias_inp_k]);
Mat bias(1, outCn, CV_32S);
Mat outputMultiplier(1, outCn, CV_32F);
bool biasOk = (float_bias.total() == (size_t)outCn);
for (int ioc = 0; ioc < outCn && biasOk; ioc++) {
float denom = inp_sc * wt_sc.at<float>(ioc);
if (std::abs(denom) < 1e-12f) { biasOk = false; break; }
float b_val = 0.f;
if (float_bias.depth() == CV_32F)
b_val = float_bias.at<float>(ioc);
else if (float_bias.depth() == CV_64F)
b_val = (float)float_bias.at<double>(ioc);
bias.at<int>(ioc) = cvRound(b_val / denom) - inp_zp * (int)cv::sum(weights.row(ioc))[0];
outputMultiplier.at<float>(ioc) = denom / out_sc_val;
}
if (biasOk) {
int firstInpDims = (int)netimpl->argData(mm_x).shape.size();
int fc_axis = std::max(1, firstInpDims - w_q.dims + 1);
LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm2, "InnerProductInt8");
fcInt8Params.set("num_output", outCn);
fcInt8Params.set("axis", fc_axis);
fcInt8Params.blobs.resize(3);
fcInt8Params.blobs[0] = weights;
fcInt8Params.blobs[1] = bias;
fcInt8Params.blobs[2] = outputMultiplier;
Ptr<Layer> fcInt8 = createFusedLayer(fcInt8Params);
if (!fcInt8.empty()) {
auto* fcInt8Layer = dynamic_cast<InnerProductLayerInt8*>(fcInt8.get());
CV_Assert(fcInt8Layer);
fcInt8Layer->input_zp = inp_zp;
fcInt8Layer->input_sc = inp_sc;
fcInt8Layer->output_zp = out_zp_i;
fcInt8Layer->output_sc = out_sc_val;
fcInt8Layer->output_type = fc_out_type;
fcInt8Layer->per_channel = per_channel;
fused_layer_idx = add_bias_idx;
newprog[add_bias_idx] = fcInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(add_bias->inputs[mm_inp_k]); // matmul out
removed_args.push_back(mm_x);
removed_args.push_back(mm_w);
newprog[mm2_idx] = Ptr<Layer>();
newprog[dq_x_idx] = Ptr<Layer>();
newprog[dq_w_idx] = Ptr<Layer>();
break;
}
}
}
}
}
}
}
}
{
ActivationLayerInt8* activ_int8 = dynamic_cast<ActivationLayerInt8*>(layer_ptr);
if (activ_int8 && ninputs == 1 &&
usecounts.at(inputs[0].idx) == 1) {
Arg activ_inp = inputs[0];
int producer_idx = producer_of.at(activ_inp.idx);
if (producer_idx >= 0 && !newprog[producer_idx].empty()) {
Layer* producer_layer = newprog[producer_idx].get();
int pool_layer_idx = -1;
PoolingLayer* pool = 0;
if (getQdqPatternContext<PoolingLayer>(layer_ptr, ninputs, inputs, producer_of,
newprog, q_data_in, out_scale, out_zp,
pool_layer_idx, pool) && pool->inputs.size() == 1) {
Arg pool_in = pool->inputs[0];
int dq_idx = producer_of.at(pool_in.idx);
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
float inp_sc = 0.f, out_sc = 0.f;
int inp_zp = 0, out_zp_i = 0;
const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U);
const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U);
if (dq && dq->inputs.size() >= 3 && pool_in_int8 && pool_out_int8 && usecounts.at(pool_in.idx) == 1) {
inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]);
inp_zp = pool_zp_m.depth() == CV_8U
? (int)pool_zp_m.at<uint8_t>(0)
: (int)pool_zp_m.at<int8_t>(0);
out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& pool_out_zp_m = netimpl->argTensor(out_zp);
out_zp_i = pool_out_zp_m.depth() == CV_8U
? (int)pool_out_zp_m.at<uint8_t>(0)
: (int)pool_out_zp_m.at<int8_t>(0);
bool isGlobalAve = pool->globalPooling;
bool isMax = !isGlobalAve;
if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) ||
(isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) {
LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "Pool2Int8");
poolInt8Params.blobs.clear();
poolInt8Params.set("input_scale", inp_sc);
poolInt8Params.set("input_zeropoint", inp_zp);
poolInt8Params.set("scales", out_sc);
poolInt8Params.set("zeropoints", out_zp_i);
poolInt8Params.set("is_max_pool", isMax);
poolInt8Params.set("global_pooling", isGlobalAve);
Ptr<Layer> poolInt8 = createFusedLayer(poolInt8Params);
if (!poolInt8.empty()) {
auto* poolInt8Layer = dynamic_cast<Pool2Int8Layer*>(poolInt8.get());
CV_Assert(poolInt8Layer);
fused_layer_idx = pool_layer_idx;
newprog[pool_layer_idx] = poolInt8;
fused_inputs.assign(1, dq->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(pool_in);
newprog[dq_idx] = Ptr<Layer>();
break;
}
}
}
}
ActivationLayerInt8* activ_int8 = dynamic_cast<ActivationLayerInt8*>(layer_ptr);
if (activ_int8 && ninputs == 1 &&
usecounts.at(inputs[0].idx) == 1) {
Arg activ_inp = inputs[0];
int producer_idx = producer_of.at(activ_inp.idx);
if (producer_idx >= 0 && !newprog[producer_idx].empty()) {
Layer* producer_layer = newprog[producer_idx].get();
float prod_sc = 0.f;
int prod_zp = 0;
if (getInt8OutputParams(producer_layer, prod_sc, prod_zp) &&
prod_sc == activ_int8->input_sc &&
prod_zp == activ_int8->input_zp) {
Ptr<ActivationLayer> activ_layer = layer.dynamicCast<ActivationLayer>();
ConvolutionLayerInt8* conv_int8 =
dynamic_cast<ConvolutionLayerInt8*>(producer_layer);
if (conv_int8 && conv_int8->output_sc == activ_int8->input_sc &&
conv_int8->output_zp == activ_int8->input_zp) {
if (newprog[producer_idx]->setActivation(activ_layer)) {
conv_int8->output_sc = activ_int8->output_sc;
conv_int8->output_zp = activ_int8->output_zp;
fused_layer_idx = producer_idx;
removed_args.push_back(activ_inp);
break;
}
}
InnerProductLayerInt8* fc_int8 =
dynamic_cast<InnerProductLayerInt8*>(producer_layer);
if (fc_int8 && fc_int8->output_sc == activ_int8->input_sc &&
fc_int8->output_zp == activ_int8->input_zp) {
if (newprog[producer_idx]->setActivation(activ_layer)) {
fc_int8->output_sc = activ_int8->output_sc;
fc_int8->output_zp = activ_int8->output_zp;
fused_layer_idx = producer_idx;
removed_args.push_back(activ_inp);
break;
}
}
EltwiseLayerInt8* elt_int8 =
dynamic_cast<EltwiseLayerInt8*>(producer_layer);
if (elt_int8 && elt_int8->output_sc == activ_int8->input_sc &&
elt_int8->output_zp == activ_int8->input_zp) {
if (newprog[producer_idx]->setActivation(activ_layer)) {
elt_int8->output_sc = activ_int8->output_sc;
elt_int8->output_zp = activ_int8->output_zp;
fused_layer_idx = producer_idx;
removed_args.push_back(activ_inp);
break;
}
if (newprog[producer_idx]->setActivation(activ_layer)) {
setInt8OutputParams(producer_layer,
activ_int8->output_sc,
activ_int8->output_zp);
fused_layer_idx = producer_idx;
removed_args.push_back(activ_inp);
break;
}
}
}
@@ -770,12 +896,13 @@ struct ModelFusionQDQ
if (fused_layer_idx >= 0) {
modified = true;
Layer* fused_layer = newprog[fused_layer_idx];
fused_layer->outputs = outputs;
const std::vector<Arg>& final_outputs = override_outputs.empty() ? outputs : override_outputs;
fused_layer->outputs = final_outputs;
if (!fused_inputs.empty())
fused_layer->inputs = fused_inputs;
for (Arg new_out: outputs)
for (const Arg& new_out: final_outputs)
producer_of[new_out.idx] = fused_layer_idx;
for (Arg old_out: removed_args) {
for (const Arg& old_out: removed_args) {
usecounts.at(old_out.idx) = 0;
producer_of.at(old_out.idx) = -1;
}
@@ -787,6 +914,41 @@ struct ModelFusionQDQ
}
if (modified) {
std::vector<int> uc(nargs, 0);
for (const auto& layer : newprog) {
if (layer.empty()) continue;
for (const Arg& inp : layer->inputs) {
if (inp.idx >= 0 && inp.idx < (int)nargs)
uc[inp.idx]++;
}
}
const auto& graph_outputs = graph->outputs();
for (const Arg& out : graph_outputs) {
if (out.idx >= 0 && out.idx < (int)nargs)
uc[out.idx]++;
}
bool changed = true;
while (changed) {
changed = false;
for (auto& layer : newprog) {
if (layer.empty()) continue;
bool all_dead = true;
for (const Arg& out : layer->outputs) {
if (out.idx >= 0 && out.idx < (int)nargs && uc[out.idx] > 0) {
all_dead = false;
break;
}
}
if (all_dead) {
for (const Arg& inp : layer->inputs) {
if (inp.idx >= 0 && inp.idx < (int)nargs)
uc[inp.idx]--;
}
layer = Ptr<Layer>();
changed = true;
}
}
}
size_t i, j = 0, newops = newprog.size();
for (i = 0; i < newops; i++) {
if (!newprog[i].empty()) {
modules/dnn/src/init.cpp
+3
View File
@@ -263,6 +263,9 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(EltwiseInt8, EltwiseLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(Conv2Int8, Conv2Int8Layer);
CV_DNN_REGISTER_LAYER_CLASS(Pool2Int8, Pool2Int8Layer);
CV_DNN_REGISTER_LAYER_CLASS(Eltwise2Int8, Eltwise2Int8Layer);
CV_DNN_REGISTER_LAYER_CLASS(BatchNormInt8, BatchNormLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(ScaleInt8, ScaleLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(ShiftInt8, ShiftLayerInt8);
modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp
+1221
View File
@@ -0,0 +1,1221 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "opencv2/core/hal/intrin.hpp"
#include "../layers/conv2_common.hpp"
#if !defined(CV_AVXVNNI_AVAILABLE)
#if (CV_TRY_AVX2 || CV_AVX2) && \
((defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 11) || \
(defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ >= 12))
#define CV_AVXVNNI_AVAILABLE 1
#else
#define CV_AVXVNNI_AVAILABLE 0
#endif
#endif
namespace cv {
namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
void convInt8Block(const void* inp_, const void* residual_,
void* out_, const ConvState& cs,
const void* weights_,
const void* weightsVNNI_,
const int* bias, const int* biasVNNI_,
const float* multiplier,
int inp_zp, int out_zp,
const int8_t* activLUT,
bool inputIsU8);
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY)
#if CV_AVXVNNI_AVAILABLE
#if defined(__x86_64__) || defined(_M_X64)
#include <immintrin.h>
#endif
#ifdef __clang__
#pragma clang attribute push(__attribute__((target("avx2,avxvnni"))), apply_to = function)
#else
#pragma GCC push_options
#pragma GCC target("avxvnni")
#endif
static void convInt8BlockVNNI(const void* inp_, const void* residual_,
void* out_, const ConvState& cs,
const void* weightsVNNI_,
const int* biasVNNI, const float* multiplier,
int inp_zp, int out_zp,
const int8_t* activLUT,
bool inputIsU8)
{
constexpr int C0 = 8, K0 = 8;
constexpr int SPAT_BLOCK_SIZE = 8;
constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS;
CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.inpshape.back() == C0);
const MatShape& inpshape = cs.inpshape;
const MatShape& outshape = cs.outshape;
int N = outshape[0];
int ndims = outshape.dims;
int K = outshape.channels();
int C = inpshape.channels();
int K1 = (K + K0 - 1) / K0;
int C1 = (C + C0 - 1) / C0;
int D_ = ndims >= 6 ? outshape[ndims-4] : 1;
int H_ = ndims >= 5 ? outshape[ndims-3] : 1;
int W_ = outshape[ndims-2];
int planeblocks_ = D_ * H_ * W_;
int ngroups = cs.ngroups;
int Kg = K / ngroups;
int Cg = C / ngroups;
int Kblk = cs.wshape[1];
int ksize_ = cs.wshape[2];
int C1Max = cs.wshape[3];
int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS];
int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1];
int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2];
const int8_t* inp = (const int8_t*)inp_;
const int8_t* residual = (const int8_t*)residual_;
int8_t* out = (int8_t*)out_;
const int8_t* wdata = (const int8_t*)weightsVNNI_;
const int* ofsZYX = cs.coordtab.data();
size_t outtotal = outshape.total();
if ((Kg % K0) != 0) memset(out, 0, outtotal);
int total_blocks = N * ngroups * Kblk;
const __m256i v_xor = inputIsU8 ? _mm256_setzero_si256() : _mm256_set1_epi8((char)0x80);
parallel_for_(Range(0, total_blocks), [&](const Range& range) {
constexpr int C0 = 8, K0 = 8;
constexpr int C0shift = 3;
int D = D_, H = H_, W = W_;
int Di = ndims >= 6 ? inpshape[ndims-4] : 1;
int Hi = ndims >= 5 ? inpshape[ndims-3] : 1;
int Wi = inpshape[ndims-2];
int planeblocks = planeblocks_;
int iplanesize = Di * Hi * Wi * C0;
int planesize = planeblocks * K0;
int Sz = cs.strides[0], Sy = cs.strides[1], Sx = cs.strides[2];
int padZ = cs.pads[0], padY = cs.pads[1], padX = cs.pads[2];
int ksize = ksize_;
int8_t zbuf[C0];
memset(zbuf, (uint8_t)inp_zp, C0);
for (int t = range.start; t < range.end; t++) {
int n = t / (ngroups * Kblk);
int rem = t - n * (ngroups * Kblk);
int g = rem / Kblk;
int kblk = rem - g * Kblk;
int k_base = g * Kg + kblk * K0;
if (k_base >= K) continue;
int c_start = g * Cg;
int c1_start = c_start >> C0shift;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) >> C0shift;
const int8_t* inpbaseptr = inp + (size_t)(n * C1 + c1_start) * iplanesize;
const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * ksize * C1Max * C0 * K0;
int k1 = k_base >> C0shift;
int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize;
const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr;
alignas(32) int32_t biasbuf[K0];
memcpy(biasbuf, biasVNNI + k_base, K0 * sizeof(int32_t));
alignas(32) float multbuf[K0];
memcpy(multbuf, multiplier + k_base, K0 * sizeof(float));
int D_l = D, H_l = H, W_l = W;
int Di_l = Di, Hi_l = Hi, Wi_l = Wi;
int iplanesize_l = iplanesize;
int planeblocks_l = planeblocks;
int ksize_l = ksize;
if (ksize == 1 && Sx == 1 && Sy == 1 && Sz == 1) {
W_l *= D_l * H_l;
Wi_l *= Di_l * Hi_l;
D_l = Di_l = H_l = Hi_l = 1;
iplanesize_l = Wi_l * C0;
planeblocks_l = W_l;
}
int p = 0;
for (; p < planeblocks_l; p += SPAT_BLOCK_SIZE) {
if (p + SPAT_BLOCK_SIZE > planeblocks_l) {
if (p == 0) break;
p = planeblocks_l - SPAT_BLOCK_SIZE;
}
Vec3i pt[SPAT_BLOCK_SIZE];
bool inner[SPAT_BLOCK_SIZE];
bool all_inner = true;
if ((p % W_l) + SPAT_BLOCK_SIZE <= W_l) {
int zj = p / (H_l * W_l);
int yxj = p - zj * H_l * W_l;
int yj = yxj / W_l;
int xj0 = yxj - yj * W_l;
bool zy_inner = (zj >= innerZ0 && zj < innerZ1) &&
(yj >= innerY0 && yj < innerY1);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int xj = xj0 + j;
pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX);
inner[j] = zy_inner && (xj >= innerX0 && xj < innerX1);
all_inner &= inner[j];
}
} else {
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int pj = p + j;
int zj = pj / (H_l * W_l);
int yxj = pj - zj * H_l * W_l;
int yj = yxj / W_l;
int xj = yxj - yj * W_l;
pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX);
inner[j] = (zj >= innerZ0 && zj < innerZ1) &&
(yj >= innerY0 && yj < innerY1) &&
(xj >= innerX0 && xj < innerX1);
all_inner &= inner[j];
}
}
__m256i vbias = _mm256_load_si256((const __m256i*)biasbuf);
__m256i s0 = vbias, s1 = vbias, s2 = vbias, s3 = vbias;
__m256i s4 = vbias, s5 = vbias, s6 = vbias, s7 = vbias;
#define CONV_VNNI_MAC(j) { \
__m256i x0 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)&inptr[(j)][0]), v_xor); \
__m256i x1 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)&inptr[(j)][4]), v_xor); \
s##j = _mm256_dpbusd_epi32(s##j, x0, wg0); \
s##j = _mm256_dpbusd_epi32(s##j, x1, wg1); \
}
if (all_inner) {
for (int i = 0; i < ksize_l; i++) {
const int8_t* inptr[SPAT_BLOCK_SIZE];
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int zij = pt[j][0] + ofsZYX[i * 3];
int yij = pt[j][1] + ofsZYX[i * 3 + 1];
int xij = pt[j][2] + ofsZYX[i * 3 + 2];
inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0;
}
const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) {
__m256i wg0 = _mm256_load_si256((const __m256i*)(wptr));
__m256i wg1 = _mm256_load_si256((const __m256i*)(wptr + 32));
CONV_VNNI_MAC(0); CONV_VNNI_MAC(1);
CONV_VNNI_MAC(2); CONV_VNNI_MAC(3);
CONV_VNNI_MAC(4); CONV_VNNI_MAC(5);
CONV_VNNI_MAC(6); CONV_VNNI_MAC(7);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++)
inptr[j] += iplanesize_l;
}
}
} else {
for (int i = 0; i < ksize_l; i++) {
const int8_t* inptr[SPAT_BLOCK_SIZE];
int inpstep[SPAT_BLOCK_SIZE];
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int zij = pt[j][0] + ofsZYX[i * 3];
int yij = pt[j][1] + ofsZYX[i * 3 + 1];
int xij = pt[j][2] + ofsZYX[i * 3 + 2];
if (inner[j] || ((((unsigned)zij < (unsigned)Di_l) &
((unsigned)yij < (unsigned)Hi_l) &
((unsigned)xij < (unsigned)Wi_l)) != 0)) {
inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0;
inpstep[j] = iplanesize_l;
} else {
inptr[j] = zbuf;
inpstep[j] = 0;
}
}
const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) {
__m256i wg0 = _mm256_load_si256((const __m256i*)(wptr));
__m256i wg1 = _mm256_load_si256((const __m256i*)(wptr + 32));
CONV_VNNI_MAC(0); CONV_VNNI_MAC(1);
CONV_VNNI_MAC(2); CONV_VNNI_MAC(3);
CONV_VNNI_MAC(4); CONV_VNNI_MAC(5);
CONV_VNNI_MAC(6); CONV_VNNI_MAC(7);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++)
inptr[j] += inpstep[j];
}
}
}
#undef CONV_VNNI_MAC
__m256 vmult = _mm256_load_ps(multbuf);
__m256 vzp = _mm256_set1_ps((float)out_zp);
#define CONV_VNNI_STORE(j) { \
__m256 facc = _mm256_add_ps( \
_mm256_mul_ps(_mm256_cvtepi32_ps(s##j), vmult), vzp); \
if (resptr) { \
__m256i res32 = _mm256_cvtepi8_epi32( \
_mm_loadl_epi64((const __m128i*)(resptr + (p + (j)) * K0))); \
facc = _mm256_add_ps(facc, \
_mm256_sub_ps(_mm256_cvtepi32_ps(res32), vzp)); \
} \
__m256i ival = _mm256_cvtps_epi32(facc); \
__m128i lo = _mm256_castsi256_si128(ival); \
__m128i hi = _mm256_extracti128_si256(ival, 1); \
__m128i packed16 = _mm_packs_epi32(lo, hi); \
__m128i packed8 = inputIsU8 ? _mm_packus_epi16(packed16, packed16) \
: _mm_packs_epi16(packed16, packed16); \
int8_t* optr = outptr + (p + (j)) * K0; \
if (activLUT) { \
alignas(16) int8_t tmp8[16]; \
_mm_store_si128((__m128i*)tmp8, packed8); \
for (int k = 0; k < K0; k++) \
optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128]; \
} else { \
_mm_storel_epi64((__m128i*)optr, packed8); \
} \
}
CONV_VNNI_STORE(0); CONV_VNNI_STORE(1);
CONV_VNNI_STORE(2); CONV_VNNI_STORE(3);
CONV_VNNI_STORE(4); CONV_VNNI_STORE(5);
CONV_VNNI_STORE(6); CONV_VNNI_STORE(7);
#undef CONV_VNNI_STORE
}
// Scalar tail
for (; p < planeblocks_l; p++) {
alignas(32) int32_t acc[K0];
memcpy(acc, biasbuf, K0 * sizeof(int32_t));
int zj = p / (H_l * W_l);
int yxj = p - zj * H_l * W_l;
int yj = yxj / W_l;
int xj = yxj - yj * W_l;
int zi_base = zj * Sz - padZ, yi_base = yj * Sy - padY, xi_base = xj * Sx - padX;
for (int i = 0; i < ksize_l; i++) {
int zi = zi_base + ofsZYX[i * 3];
int yi = yi_base + ofsZYX[i * 3 + 1];
int xi = xi_base + ofsZYX[i * 3 + 2];
if ((((unsigned)zi >= (unsigned)Di_l) | ((unsigned)yi >= (unsigned)Hi_l) |
((unsigned)xi >= (unsigned)Wi_l)) != 0) continue;
const int8_t* inptr = inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0;
const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1i = 0; c1i < cblocks; c1i++, inptr += iplanesize_l, wptr += C0 * K0) {
for (int c0 = 0; c0 < C0; c0++) {
int ival = inputIsU8 ? (int)(uint8_t)inptr[c0]
: (int)(uint8_t)((uint8_t)inptr[c0] ^ 0x80u);
const int8_t* wp = wptr + c0 * K0;
for (int k0 = 0; k0 < K0; k0++)
acc[k0] += ival * (int)wp[k0];
}
}
}
int8_t* optr = outptr + p * K0;
const int8_t* rptr = resptr ? resptr + p * K0 : nullptr;
for (int k0 = 0; k0 < K0; k0++) {
float val = (float)acc[k0] * multbuf[k0] + (float)out_zp;
if (rptr) val += (float)((int)rptr[k0] - out_zp);
int ival = cvRound(val);
if (inputIsU8) {
ival = std::max(0, std::min(255, ival));
if (activLUT) ival = (int)(uint8_t)activLUT[ival];
((uint8_t*)optr)[k0] = (uint8_t)ival;
} else {
ival = std::max(-128, std::min(127, ival));
if (activLUT) ival = (int)activLUT[ival + 128];
optr[k0] = (int8_t)ival;
}
}
}
}
});
}
#ifdef __clang__
#pragma clang attribute pop
#else
#pragma GCC pop_options
#endif
#endif // CV_AVXVNNI_AVAILABLE
#if CV_NEON && defined(CV_NEON_DOT) && CV_NEON_DOT
#include <arm_neon.h>
#define CONV_NEON_STORE(j) { \
float32x4_t facc_lo = vfmaq_f32(vzp, vcvtq_f32_s32(s##j##_lo), vmult_lo); \
float32x4_t facc_hi = vfmaq_f32(vzp, vcvtq_f32_s32(s##j##_hi), vmult_hi); \
if (resptr) { \
const int8_t* rp = resptr + (p + (j)) * K0; \
if (inputIsU8) { \
uint16x8_t r16u = vmovl_u8(vld1_u8((const uint8_t*)rp)); \
facc_lo = vaddq_f32(facc_lo, vsubq_f32( \
vcvtq_f32_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(r16u)))), vzp)); \
facc_hi = vaddq_f32(facc_hi, vsubq_f32( \
vcvtq_f32_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(r16u)))), vzp)); \
} else { \
int16x8_t r16 = vmovl_s8(vld1_s8(rp)); \
facc_lo = vaddq_f32(facc_lo, vsubq_f32( \
vcvtq_f32_s32(vmovl_s16(vget_low_s16(r16))), vzp)); \
facc_hi = vaddq_f32(facc_hi, vsubq_f32( \
vcvtq_f32_s32(vmovl_s16(vget_high_s16(r16))), vzp)); \
} \
} \
int32x4_t ival_lo = vcvtnq_s32_f32(facc_lo); \
int32x4_t ival_hi = vcvtnq_s32_f32(facc_hi); \
int16x4_t p16_lo = vqmovn_s32(ival_lo); \
int16x4_t p16_hi = vqmovn_s32(ival_hi); \
int16x8_t p16 = vcombine_s16(p16_lo, p16_hi); \
int8_t* optr = outptr + (p + (j)) * K0; \
if (inputIsU8) { \
uint8x8_t p8u = vqmovun_s16(p16); \
if (activLUT) { \
uint8_t tmp8[8]; \
vst1_u8(tmp8, p8u); \
for (int k = 0; k < K0; k++) \
((uint8_t*)optr)[k] = (uint8_t)activLUT[tmp8[k]]; \
} else { \
vst1_u8((uint8_t*)optr, p8u); \
} \
} else { \
int8x8_t p8 = vqmovn_s16(p16); \
if (activLUT) { \
int8_t tmp8[8]; \
vst1_s8(tmp8, p8); \
for (int k = 0; k < K0; k++) \
optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128]; \
} else { \
vst1_s8(optr, p8); \
} \
} \
}
static void convInt8BlockNEON(const void* inp_, const void* residual_,
void* out_, const ConvState& cs,
const void* weightsVNNI_,
const int* bias, const float* multiplier,
int inp_zp, int out_zp,
const int8_t* activLUT,
bool inputIsU8)
{
constexpr int C0 = 8, K0 = 8;
constexpr int C0shift = 3;
constexpr int SPAT_BLOCK_SIZE = 8;
constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS;
CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.inpshape.back() == C0);
const MatShape& inpshape = cs.inpshape;
const MatShape& outshape = cs.outshape;
int N = outshape[0];
int ndims = outshape.dims;
int K = outshape.channels();
int C = inpshape.channels();
int K1 = (K + K0 - 1) / K0;
int C1 = (C + C0 - 1) / C0;
int D_ = ndims >= 6 ? outshape[ndims-4] : 1;
int H_ = ndims >= 5 ? outshape[ndims-3] : 1;
int W_ = outshape[ndims-2];
int planeblocks_ = D_ * H_ * W_;
int ngroups = cs.ngroups;
int Kg = K / ngroups;
int Cg = C / ngroups;
int Kblk = cs.wshape[1];
int ksize_ = cs.wshape[2];
int C1Max = cs.wshape[3];
const int8_t* inp = (const int8_t*)inp_;
const int8_t* residual = (const int8_t*)residual_;
int8_t* out = (int8_t*)out_;
const int8_t* wdata = (const int8_t*)weightsVNNI_;
const int* ofsZYX = cs.coordtab.data();
size_t outtotal = outshape.total();
if ((Kg % K0) != 0) memset(out, 0, outtotal);
const int8x16_t v_xor = inputIsU8 ? vdupq_n_s8((int8_t)0x80) : vdupq_n_s8(0);
int Sz = cs.strides[0], Sy = cs.strides[1], Sx = cs.strides[2];
if (ksize_ == 1 && Sx == 1 && Sy == 1 && Sz == 1) {
int total_spatial = planeblocks_;
int iplanesize = total_spatial * C0;
int cblocks_g = (Cg + C0 - 1) / C0;
int cblk_bytes = C0 * cblocks_g;
int SPAT_TILE = std::max(SPAT_BLOCK_SIZE,
((48 * 1024) / std::max(1, cblk_bytes)) & ~(SPAT_BLOCK_SIZE - 1));
SPAT_TILE = std::min(SPAT_TILE, total_spatial);
SPAT_TILE = std::max(SPAT_TILE, SPAT_BLOCK_SIZE);
int wblk_bytes = C0 * K0 * cblocks_g;
int KG = std::min(Kblk, std::max(1, (48 * 1024) / std::max(1, wblk_bytes)));
int ntiles = (total_spatial + SPAT_TILE - 1) / SPAT_TILE;
int nktiles = (Kblk + KG - 1) / KG;
while (N * ngroups * ntiles * nktiles < 16 && KG > 1) {
KG = std::max(1, KG / 2);
nktiles = (Kblk + KG - 1) / KG;
}
int total_tasks = N * ngroups * ntiles * nktiles;
parallel_for_(Range(0, total_tasks), [&](const Range& range) {
for (int task = range.start; task < range.end; task++) {
int temp = task;
int n = temp / (ngroups * ntiles * nktiles);
temp -= n * (ngroups * ntiles * nktiles);
int g = temp / (ntiles * nktiles);
temp -= g * (ntiles * nktiles);
int tile_idx = temp / nktiles;
int kgroup_idx = temp - tile_idx * nktiles;
int p_start = tile_idx * SPAT_TILE;
int p_end = std::min(p_start + SPAT_TILE, total_spatial);
int kblk_start = kgroup_idx * KG;
int kblk_end = std::min(kblk_start + KG, Kblk);
int c_start = g * Cg;
int c1_start = c_start >> C0shift;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) >> C0shift;
const int8_t* inpbase = inp + (size_t)(n * C1 + c1_start) * iplanesize;
int planesize = total_spatial * K0;
for (int kblk = kblk_start; kblk < kblk_end; kblk++) {
int k_base = g * Kg + kblk * K0;
if (k_base >= K) continue;
int k1 = k_base >> C0shift;
int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize;
const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr;
const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * C1Max * C0 * K0;
int32x4_t vbias_lo = vld1q_s32(bias + k_base);
int32x4_t vbias_hi = vld1q_s32(bias + k_base + 4);
float32x4_t vmult_lo = vld1q_f32(multiplier + k_base);
float32x4_t vmult_hi = vld1q_f32(multiplier + k_base + 4);
float32x4_t vzp = vdupq_n_f32((float)out_zp);
int p = p_start;
for (; p <= p_end - SPAT_BLOCK_SIZE; p += SPAT_BLOCK_SIZE) {
int32x4_t s0_lo = vbias_lo, s0_hi = vbias_hi;
int32x4_t s1_lo = vbias_lo, s1_hi = vbias_hi;
int32x4_t s2_lo = vbias_lo, s2_hi = vbias_hi;
int32x4_t s3_lo = vbias_lo, s3_hi = vbias_hi;
int32x4_t s4_lo = vbias_lo, s4_hi = vbias_hi;
int32x4_t s5_lo = vbias_lo, s5_hi = vbias_hi;
int32x4_t s6_lo = vbias_lo, s6_hi = vbias_hi;
int32x4_t s7_lo = vbias_lo, s7_hi = vbias_hi;
const int8_t* inpptr = inpbase + (size_t)p * C0;
const int8_t* wptr = wbaseptr;
for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) {
int8x16_t wg0 = vld1q_s8(wptr);
int8x16_t wg1 = vld1q_s8(wptr + 16);
int8x16_t wg2 = vld1q_s8(wptr + 32);
int8x16_t wg3 = vld1q_s8(wptr + 48);
const int8_t* ip = inpptr;
#define CONV_NEON_MAC_1x1(j) { \
int8x16_t x_lo = veorq_s8(vreinterpretq_s8_s32( \
vdupq_n_s32(*(const int32_t*)&ip[0])), v_xor); \
int8x16_t x_hi = veorq_s8(vreinterpretq_s8_s32( \
vdupq_n_s32(*(const int32_t*)&ip[4])), v_xor); \
s##j##_lo = vdotq_s32(s##j##_lo, x_lo, wg0); \
s##j##_hi = vdotq_s32(s##j##_hi, x_lo, wg1); \
s##j##_lo = vdotq_s32(s##j##_lo, x_hi, wg2); \
s##j##_hi = vdotq_s32(s##j##_hi, x_hi, wg3); \
ip += C0; \
}
CONV_NEON_MAC_1x1(0); CONV_NEON_MAC_1x1(1);
CONV_NEON_MAC_1x1(2); CONV_NEON_MAC_1x1(3);
CONV_NEON_MAC_1x1(4); CONV_NEON_MAC_1x1(5);
CONV_NEON_MAC_1x1(6); CONV_NEON_MAC_1x1(7);
#undef CONV_NEON_MAC_1x1
inpptr += iplanesize;
}
CONV_NEON_STORE(0); CONV_NEON_STORE(1);
CONV_NEON_STORE(2); CONV_NEON_STORE(3);
CONV_NEON_STORE(4); CONV_NEON_STORE(5);
CONV_NEON_STORE(6); CONV_NEON_STORE(7);
}
if (p < p_end && p_end >= SPAT_BLOCK_SIZE) {
p = p_end - SPAT_BLOCK_SIZE;
int32x4_t s0_lo = vbias_lo, s0_hi = vbias_hi;
int32x4_t s1_lo = vbias_lo, s1_hi = vbias_hi;
int32x4_t s2_lo = vbias_lo, s2_hi = vbias_hi;
int32x4_t s3_lo = vbias_lo, s3_hi = vbias_hi;
int32x4_t s4_lo = vbias_lo, s4_hi = vbias_hi;
int32x4_t s5_lo = vbias_lo, s5_hi = vbias_hi;
int32x4_t s6_lo = vbias_lo, s6_hi = vbias_hi;
int32x4_t s7_lo = vbias_lo, s7_hi = vbias_hi;
const int8_t* inpptr = inpbase + (size_t)p * C0;
const int8_t* wptr = wbaseptr;
for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) {
int8x16_t wg0 = vld1q_s8(wptr);
int8x16_t wg1 = vld1q_s8(wptr + 16);
int8x16_t wg2 = vld1q_s8(wptr + 32);
int8x16_t wg3 = vld1q_s8(wptr + 48);
const int8_t* ip = inpptr;
#define CONV_NEON_MAC_T(j) { \
int8x16_t x_lo = veorq_s8(vreinterpretq_s8_s32( \
vdupq_n_s32(*(const int32_t*)&ip[0])), v_xor); \
int8x16_t x_hi = veorq_s8(vreinterpretq_s8_s32( \
vdupq_n_s32(*(const int32_t*)&ip[4])), v_xor); \
s##j##_lo = vdotq_s32(s##j##_lo, x_lo, wg0); \
s##j##_hi = vdotq_s32(s##j##_hi, x_lo, wg1); \
s##j##_lo = vdotq_s32(s##j##_lo, x_hi, wg2); \
s##j##_hi = vdotq_s32(s##j##_hi, x_hi, wg3); \
ip += C0; \
}
CONV_NEON_MAC_T(0); CONV_NEON_MAC_T(1);
CONV_NEON_MAC_T(2); CONV_NEON_MAC_T(3);
CONV_NEON_MAC_T(4); CONV_NEON_MAC_T(5);
CONV_NEON_MAC_T(6); CONV_NEON_MAC_T(7);
#undef CONV_NEON_MAC_T
inpptr += iplanesize;
}
CONV_NEON_STORE(0); CONV_NEON_STORE(1);
CONV_NEON_STORE(2); CONV_NEON_STORE(3);
CONV_NEON_STORE(4); CONV_NEON_STORE(5);
CONV_NEON_STORE(6); CONV_NEON_STORE(7);
}
}
}
});
return;
}
int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS];
int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1];
int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2];
int total_blocks = N * ngroups * Kblk;
parallel_for_(Range(0, total_blocks), [&](const Range& range) {
int H = H_, W = W_;
int Di = ndims >= 6 ? inpshape[ndims-4] : 1;
int Hi = ndims >= 5 ? inpshape[ndims-3] : 1;
int Wi = inpshape[ndims-2];
int planeblocks = planeblocks_;
int iplanesize = Di * Hi * Wi * C0;
int planesize = planeblocks * K0;
int padZ = cs.pads[0], padY = cs.pads[1], padX = cs.pads[2];
int ksize = ksize_;
int8_t zbuf[C0];
memset(zbuf, (uint8_t)inp_zp, C0);
for (int t = range.start; t < range.end; t++) {
int n = t / (ngroups * Kblk);
int rem = t - n * (ngroups * Kblk);
int g = rem / Kblk;
int kblk = rem - g * Kblk;
int k_base = g * Kg + kblk * K0;
if (k_base >= K) continue;
int c_start = g * Cg;
int c1_start = c_start >> C0shift;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) >> C0shift;
const int8_t* inpbaseptr = inp + (size_t)(n * C1 + c1_start) * iplanesize;
const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * ksize * C1Max * C0 * K0;
int k1 = k_base >> C0shift;
int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize;
const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr;
int32x4_t vbias_lo = vld1q_s32(bias + k_base);
int32x4_t vbias_hi = vld1q_s32(bias + k_base + 4);
float32x4_t vmult_lo = vld1q_f32(multiplier + k_base);
float32x4_t vmult_hi = vld1q_f32(multiplier + k_base + 4);
float32x4_t vzp = vdupq_n_f32((float)out_zp);
int H_l = H, W_l = W;
int Di_l = Di, Hi_l = Hi, Wi_l = Wi;
int iplanesize_l = iplanesize;
int planeblocks_l = planeblocks;
int ksize_l = ksize;
int p = 0;
for (; p < planeblocks_l; p += SPAT_BLOCK_SIZE) {
if (p + SPAT_BLOCK_SIZE > planeblocks_l) {
if (p == 0) break;
p = planeblocks_l - SPAT_BLOCK_SIZE;
}
Vec3i pt[SPAT_BLOCK_SIZE];
bool inner[SPAT_BLOCK_SIZE];
bool all_inner = true;
if ((p % W_l) + SPAT_BLOCK_SIZE <= W_l) {
int zj = p / (H_l * W_l);
int yxj = p - zj * H_l * W_l;
int yj = yxj / W_l;
int xj0 = yxj - yj * W_l;
bool zy_inner = (zj >= innerZ0 && zj < innerZ1) &&
(yj >= innerY0 && yj < innerY1);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int xj = xj0 + j;
pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX);
inner[j] = zy_inner && (xj >= innerX0 && xj < innerX1);
all_inner &= inner[j];
}
} else {
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int pj = p + j;
int zj = pj / (H_l * W_l);
int yxj = pj - zj * H_l * W_l;
int yj = yxj / W_l;
int xj = yxj - yj * W_l;
pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX);
inner[j] = (zj >= innerZ0 && zj < innerZ1) &&
(yj >= innerY0 && yj < innerY1) &&
(xj >= innerX0 && xj < innerX1);
all_inner &= inner[j];
}
}
int32x4_t s0_lo = vbias_lo, s0_hi = vbias_hi;
int32x4_t s1_lo = vbias_lo, s1_hi = vbias_hi;
int32x4_t s2_lo = vbias_lo, s2_hi = vbias_hi;
int32x4_t s3_lo = vbias_lo, s3_hi = vbias_hi;
int32x4_t s4_lo = vbias_lo, s4_hi = vbias_hi;
int32x4_t s5_lo = vbias_lo, s5_hi = vbias_hi;
int32x4_t s6_lo = vbias_lo, s6_hi = vbias_hi;
int32x4_t s7_lo = vbias_lo, s7_hi = vbias_hi;
#define CONV_NEON_MAC(j) { \
int8x16_t x_lo = veorq_s8(vreinterpretq_s8_s32( \
vdupq_n_s32(*(const int32_t*)&inptr[(j)][0])), v_xor); \
int8x16_t x_hi = veorq_s8(vreinterpretq_s8_s32( \
vdupq_n_s32(*(const int32_t*)&inptr[(j)][4])), v_xor); \
s##j##_lo = vdotq_s32(s##j##_lo, x_lo, wg0); \
s##j##_hi = vdotq_s32(s##j##_hi, x_lo, wg1); \
s##j##_lo = vdotq_s32(s##j##_lo, x_hi, wg2); \
s##j##_hi = vdotq_s32(s##j##_hi, x_hi, wg3); \
}
if (all_inner) {
for (int i = 0; i < ksize_l; i++) {
const int8_t* inptr[SPAT_BLOCK_SIZE];
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int zij = pt[j][0] + ofsZYX[i * 3];
int yij = pt[j][1] + ofsZYX[i * 3 + 1];
int xij = pt[j][2] + ofsZYX[i * 3 + 2];
inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0;
}
const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) {
int8x16_t wg0 = vld1q_s8(wptr);
int8x16_t wg1 = vld1q_s8(wptr + 16);
int8x16_t wg2 = vld1q_s8(wptr + 32);
int8x16_t wg3 = vld1q_s8(wptr + 48);
CONV_NEON_MAC(0); CONV_NEON_MAC(1);
CONV_NEON_MAC(2); CONV_NEON_MAC(3);
CONV_NEON_MAC(4); CONV_NEON_MAC(5);
CONV_NEON_MAC(6); CONV_NEON_MAC(7);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++)
inptr[j] += iplanesize_l;
}
}
} else {
for (int i = 0; i < ksize_l; i++) {
const int8_t* inptr[SPAT_BLOCK_SIZE];
int inpstep[SPAT_BLOCK_SIZE];
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int zij = pt[j][0] + ofsZYX[i * 3];
int yij = pt[j][1] + ofsZYX[i * 3 + 1];
int xij = pt[j][2] + ofsZYX[i * 3 + 2];
if (inner[j] || ((((unsigned)zij < (unsigned)Di_l) &
((unsigned)yij < (unsigned)Hi_l) &
((unsigned)xij < (unsigned)Wi_l)) != 0)) {
inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0;
inpstep[j] = iplanesize_l;
} else {
inptr[j] = zbuf;
inpstep[j] = 0;
}
}
const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) {
int8x16_t wg0 = vld1q_s8(wptr);
int8x16_t wg1 = vld1q_s8(wptr + 16);
int8x16_t wg2 = vld1q_s8(wptr + 32);
int8x16_t wg3 = vld1q_s8(wptr + 48);
CONV_NEON_MAC(0); CONV_NEON_MAC(1);
CONV_NEON_MAC(2); CONV_NEON_MAC(3);
CONV_NEON_MAC(4); CONV_NEON_MAC(5);
CONV_NEON_MAC(6); CONV_NEON_MAC(7);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++)
inptr[j] += inpstep[j];
}
}
}
#undef CONV_NEON_MAC
CONV_NEON_STORE(0); CONV_NEON_STORE(1);
CONV_NEON_STORE(2); CONV_NEON_STORE(3);
CONV_NEON_STORE(4); CONV_NEON_STORE(5);
CONV_NEON_STORE(6); CONV_NEON_STORE(7);
}
for (; p < planeblocks_l; p++) {
alignas(16) int32_t acc[K0];
memcpy(acc, bias + k_base, K0 * sizeof(int32_t));
int zj = p / (H_l * W_l);
int yxj = p - zj * H_l * W_l;
int yj = yxj / W_l;
int xj = yxj - yj * W_l;
int zi_base = zj * Sz - padZ, yi_base = yj * Sy - padY, xi_base = xj * Sx - padX;
for (int i = 0; i < ksize_l; i++) {
int zi = zi_base + ofsZYX[i * 3];
int yi = yi_base + ofsZYX[i * 3 + 1];
int xi = xi_base + ofsZYX[i * 3 + 2];
if ((((unsigned)zi >= (unsigned)Di_l) | ((unsigned)yi >= (unsigned)Hi_l) |
((unsigned)xi >= (unsigned)Wi_l)) != 0) continue;
const int8_t* inptr = inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0;
const int8_t* wptr_vnni = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1i = 0; c1i < cblocks; c1i++, inptr += iplanesize_l) {
const int8_t* wp = wptr_vnni + (size_t)c1i * C0 * K0;
for (int c0 = 0; c0 < C0; c0++) {
int ival = inputIsU8 ? ((int)(uint8_t)inptr[c0] - 128)
: (int)inptr[c0];
int base = (c0 < 4) ? 0 : 32;
int c_in_group = c0 & 3;
for (int k0 = 0; k0 < K0; k0++)
acc[k0] += ival * (int)wp[base + k0 * 4 + c_in_group];
}
}
}
int8_t* optr = outptr + p * K0;
const int8_t* rptr = resptr ? resptr + p * K0 : nullptr;
for (int k0 = 0; k0 < K0; k0++) {
float val = (float)acc[k0] * multiplier[k_base + k0] + (float)out_zp;
if (rptr) {
int rval = inputIsU8 ? (int)(uint8_t)rptr[k0] : (int)rptr[k0];
val += (float)(rval - out_zp);
}
int ival = cvRound(val);
if (inputIsU8) {
ival = std::max(0, std::min(255, ival));
if (activLUT) ival = (int)(uint8_t)activLUT[ival];
((uint8_t*)optr)[k0] = (uint8_t)ival;
} else {
ival = std::max(-128, std::min(127, ival));
if (activLUT) ival = (int)activLUT[ival + 128];
optr[k0] = (int8_t)ival;
}
}
}
}
});
}
#undef CONV_NEON_STORE
#endif // CV_NEON && CV_NEON_DOT
void convInt8Block(const void* inp_, const void* residual_,
void* out_, const ConvState& cs,
const void* weights_,
const void* weightsVNNI_,
const int* bias, const int* biasVNNI_,
const float* multiplier,
int inp_zp, int out_zp,
const int8_t* activLUT,
bool inputIsU8)
{
#if CV_AVXVNNI_AVAILABLE
if (cv::checkHardwareSupport(CV_CPU_AVX_VNNI) && weightsVNNI_ && biasVNNI_) {
convInt8BlockVNNI(inp_, residual_, out_, cs, weightsVNNI_,
biasVNNI_, multiplier, inp_zp, out_zp, activLUT, inputIsU8);
return;
}
#endif
#if CV_NEON && defined(CV_NEON_DOT) && CV_NEON_DOT
if (weightsVNNI_) {
convInt8BlockNEON(inp_, residual_, out_, cs, weightsVNNI_,
bias, multiplier, inp_zp, out_zp, activLUT, inputIsU8);
return;
}
#endif
#if !CV_AVXVNNI_AVAILABLE && !(CV_NEON && defined(CV_NEON_DOT) && CV_NEON_DOT)
(void)weightsVNNI_;
(void)biasVNNI_;
#else
(void)biasVNNI_;
#endif
constexpr int C0 = 8, K0 = 8;
CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.inpshape.back() == C0);
const MatShape& inpshape = cs.inpshape;
const MatShape& outshape = cs.outshape;
int N = outshape[0];
int ndims = outshape.dims;
int K = outshape.channels();
int C = inpshape.channels();
int K1 = (K + K0 - 1) / K0;
int C1 = (C + C0 - 1) / C0;
int D_ = ndims >= 6 ? outshape[ndims-4] : 1;
int H_ = ndims >= 5 ? outshape[ndims-3] : 1;
int W_ = outshape[ndims-2];
int planeblocks_ = D_ * H_ * W_;
int ngroups = cs.ngroups;
int Kg = K / ngroups;
int Cg = C / ngroups;
int Kblk = cs.wshape[1];
int ksize_ = cs.wshape[2];
int C1Max = cs.wshape[3];
#if CV_AVX2
constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS;
int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS];
int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1];
int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2];
#endif
const int8_t* inp = (const int8_t*)inp_;
const int8_t* residual = (const int8_t*)residual_;
int8_t* out = (int8_t*)out_;
const int8_t* wdata = (const int8_t*)weights_;
const int* ofsZYX = cs.coordtab.data();
size_t outtotal = outshape.total();
if ((Kg % K0) != 0) {
memset(out, 0, outtotal);
}
int total_blocks = N * ngroups * Kblk;
parallel_for_(Range(0, total_blocks), [&](const Range& range) {
constexpr int C0 = 8, K0 = 8;
constexpr int C0shift = 3;
#if CV_AVX2
constexpr int SPAT_BLOCK_SIZE = 8;
alignas(32) int32_t sumbuf[SPAT_BLOCK_SIZE * K0];
#endif
const uint8_t xor_val = inputIsU8 ? 0x80 : 0;
int D = D_, H = H_, W = W_;
int Di = ndims >= 6 ? inpshape[ndims-4] : 1;
int Hi = ndims >= 5 ? inpshape[ndims-3] : 1;
int Wi = inpshape[ndims-2];
int planeblocks = planeblocks_;
int iplanesize = Di * Hi * Wi * C0;
int planesize = planeblocks * K0;
int Sz = cs.strides[0], Sy = cs.strides[1], Sx = cs.strides[2];
int padZ = cs.pads[0], padY = cs.pads[1], padX = cs.pads[2];
int ksize = ksize_;
int8_t zbuf[C0];
memset(zbuf, (uint8_t)inp_zp, C0);
for (int t = range.start; t < range.end; t++) {
int n = t / (ngroups * Kblk);
int rem = t - n * (ngroups * Kblk);
int g = rem / Kblk;
int kblk = rem - g * Kblk;
int k_base = g * Kg + kblk * K0;
if (k_base >= K) continue;
int c_start = g * Cg;
int c1_start = c_start >> C0shift;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) >> C0shift;
const int8_t* inpbaseptr = inp + (size_t)(n * C1 + c1_start) * iplanesize;
const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * ksize * C1Max * C0 * K0;
int k1 = k_base >> C0shift;
int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize;
const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr;
int D_l = D, H_l = H, W_l = W;
int Di_l = Di, Hi_l = Hi, Wi_l = Wi;
int iplanesize_l = iplanesize;
int planeblocks_l = planeblocks;
int ksize_l = ksize;
if (ksize == 1 && Sx == 1 && Sy == 1 && Sz == 1) {
W_l *= D_l * H_l;
Wi_l *= Di_l * Hi_l;
D_l = Di_l = H_l = Hi_l = 1;
iplanesize_l = Wi_l * C0;
planeblocks_l = W_l;
}
alignas(32) int32_t biasbuf[K0];
memcpy(biasbuf, bias + k_base, K0 * sizeof(int32_t));
alignas(32) float multbuf[K0];
memcpy(multbuf, multiplier + k_base, K0 * sizeof(float));
#if CV_AVX2
int p = 0;
for (; p < planeblocks_l; p += SPAT_BLOCK_SIZE) {
if (p + SPAT_BLOCK_SIZE > planeblocks_l) {
if (p == 0) break;
p = planeblocks_l - SPAT_BLOCK_SIZE;
}
Vec3i pt[SPAT_BLOCK_SIZE];
bool inner[SPAT_BLOCK_SIZE];
if ((p % W_l) + SPAT_BLOCK_SIZE <= W_l) {
int zj = p / (H_l * W_l);
int yxj = p - zj * H_l * W_l;
int yj = yxj / W_l;
int xj0 = yxj - yj * W_l;
bool zy_inner = (zj >= innerZ0 && zj < innerZ1) &&
(yj >= innerY0 && yj < innerY1);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int xj = xj0 + j;
pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX);
inner[j] = zy_inner && (xj >= innerX0 && xj < innerX1);
}
} else {
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int pj = p + j;
int zj = pj / (H_l * W_l);
int yxj = pj - zj * H_l * W_l;
int yj = yxj / W_l;
int xj = yxj - yj * W_l;
pt[j] = Vec3i(zj * Sz - padZ, yj * Sy - padY, xj * Sx - padX);
inner[j] = (zj >= innerZ0 && zj < innerZ1) &&
(yj >= innerY0 && yj < innerY1) &&
(xj >= innerX0 && xj < innerX1);
}
}
__m256i vbias = _mm256_load_si256((const __m256i*)biasbuf);
__m256i s0 = vbias, s1 = vbias, s2 = vbias, s3 = vbias;
__m256i s4 = vbias, s5 = vbias, s6 = vbias, s7 = vbias;
for (int i = 0; i < ksize_l; i++) {
const int8_t* inptr[SPAT_BLOCK_SIZE];
int inpstep[SPAT_BLOCK_SIZE];
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
int zij = pt[j][0] + ofsZYX[i * 3];
int yij = pt[j][1] + ofsZYX[i * 3 + 1];
int xij = pt[j][2] + ofsZYX[i * 3 + 2];
if (inner[j] || ((((unsigned)zij < (unsigned)Di_l) &
((unsigned)yij < (unsigned)Hi_l) &
((unsigned)xij < (unsigned)Wi_l)) != 0)) {
inptr[j] = inpbaseptr + (((zij * Hi_l) + yij) * Wi_l + xij) * C0;
inpstep[j] = iplanesize_l;
} else {
inptr[j] = zbuf;
inpstep[j] = 0;
}
}
const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) {
#define CONV_INT8_LOAD(j, c0) \
_mm256_set1_epi32( \
(uint16_t)(int16_t)(int8_t)((uint8_t)inptr[j][(c0)] ^ xor_val) | \
((uint32_t)(uint16_t)(int16_t)(int8_t)((uint8_t)inptr[j][(c0)+1] ^ xor_val) << 16))
#define CONV_INT8_MAC_PAIR(c0) { \
__m128i wraw = _mm_loadu_si128((const __m128i*)(wptr + (c0) * K0)); \
__m128i w_intlv = _mm_unpacklo_epi8(wraw, _mm_srli_si128(wraw, 8)); \
__m256i w16 = _mm256_cvtepi8_epi16(w_intlv); \
__m256i x0 = CONV_INT8_LOAD(0, c0); \
__m256i x1 = CONV_INT8_LOAD(1, c0); \
__m256i x2 = CONV_INT8_LOAD(2, c0); \
__m256i x3 = CONV_INT8_LOAD(3, c0); \
__m256i x4 = CONV_INT8_LOAD(4, c0); \
__m256i x5 = CONV_INT8_LOAD(5, c0); \
__m256i x6 = CONV_INT8_LOAD(6, c0); \
__m256i x7 = CONV_INT8_LOAD(7, c0); \
s0 = _mm256_add_epi32(s0, _mm256_madd_epi16(x0, w16)); \
s1 = _mm256_add_epi32(s1, _mm256_madd_epi16(x1, w16)); \
s2 = _mm256_add_epi32(s2, _mm256_madd_epi16(x2, w16)); \
s3 = _mm256_add_epi32(s3, _mm256_madd_epi16(x3, w16)); \
s4 = _mm256_add_epi32(s4, _mm256_madd_epi16(x4, w16)); \
s5 = _mm256_add_epi32(s5, _mm256_madd_epi16(x5, w16)); \
s6 = _mm256_add_epi32(s6, _mm256_madd_epi16(x6, w16)); \
s7 = _mm256_add_epi32(s7, _mm256_madd_epi16(x7, w16)); \
}
CONV_INT8_MAC_PAIR(0);
CONV_INT8_MAC_PAIR(2);
CONV_INT8_MAC_PAIR(4);
CONV_INT8_MAC_PAIR(6);
#undef CONV_INT8_MAC_PAIR
#undef CONV_INT8_LOAD
for (int j = 0; j < SPAT_BLOCK_SIZE; j++)
inptr[j] += inpstep[j];
}
}
// Store accumulators
_mm256_store_si256((__m256i*)(sumbuf + 0*K0), s0);
_mm256_store_si256((__m256i*)(sumbuf + 1*K0), s1);
_mm256_store_si256((__m256i*)(sumbuf + 2*K0), s2);
_mm256_store_si256((__m256i*)(sumbuf + 3*K0), s3);
_mm256_store_si256((__m256i*)(sumbuf + 4*K0), s4);
_mm256_store_si256((__m256i*)(sumbuf + 5*K0), s5);
_mm256_store_si256((__m256i*)(sumbuf + 6*K0), s6);
_mm256_store_si256((__m256i*)(sumbuf + 7*K0), s7);
__m256 vmult = _mm256_load_ps(multbuf);
__m256 vzp = _mm256_set1_ps((float)out_zp);
for (int j = 0; j < SPAT_BLOCK_SIZE; j++) {
__m256i acc = _mm256_load_si256((const __m256i*)(sumbuf + j * K0));
__m256 facc = _mm256_cvtepi32_ps(acc);
facc = _mm256_add_ps(_mm256_mul_ps(facc, vmult), vzp);
if (resptr) {
const int8_t* rptr = resptr + (p + j) * K0;
__m256i res32 = inputIsU8 ?
_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)rptr)) :
_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)rptr));
facc = _mm256_add_ps(facc,
_mm256_sub_ps(_mm256_cvtepi32_ps(res32), vzp));
}
__m256i ival = _mm256_cvtps_epi32(facc);
__m128i lo = _mm256_castsi256_si128(ival);
__m128i hi = _mm256_extracti128_si256(ival, 1);
__m128i packed16 = _mm_packs_epi32(lo, hi);
__m128i packed8 = inputIsU8 ? _mm_packus_epi16(packed16, packed16)
: _mm_packs_epi16(packed16, packed16);
int8_t* optr = outptr + (p + j) * K0;
if (activLUT) {
alignas(16) int8_t tmp8[16];
_mm_store_si128((__m128i*)tmp8, packed8);
for (int k = 0; k < K0; k++)
optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128];
} else {
_mm_storel_epi64((__m128i*)optr, packed8);
}
}
}
for (; p < planeblocks_l; p++)
#else
for (int p = 0; p < planeblocks_l; p++)
#endif
{
alignas(32) int32_t acc[K0];
memcpy(acc, biasbuf, K0 * sizeof(int32_t));
int zj = p / (H_l * W_l);
int yxj = p - zj * H_l * W_l;
int yj = yxj / W_l;
int xj = yxj - yj * W_l;
int zi_base = zj * Sz - padZ;
int yi_base = yj * Sy - padY;
int xi_base = xj * Sx - padX;
for (int i = 0; i < ksize_l; i++) {
int zi = zi_base + ofsZYX[i * 3];
int yi = yi_base + ofsZYX[i * 3 + 1];
int xi = xi_base + ofsZYX[i * 3 + 2];
bool out_of_bounds = (((unsigned)zi >= (unsigned)Di_l) |
((unsigned)yi >= (unsigned)Hi_l) |
((unsigned)xi >= (unsigned)Wi_l)) != 0;
const int8_t* inptr = out_of_bounds ? zbuf :
inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0;
int inpstep = out_of_bounds ? 0 : iplanesize_l;
const int8_t* wptr = wbaseptr + (size_t)i * C1Max * K0 * C0;
for (int c1 = 0; c1 < cblocks; c1++, inptr += inpstep, wptr += C0 * K0) {
for (int c0 = 0; c0 < C0; c0++) {
int ival = (int)(int8_t)((uint8_t)inptr[c0] ^ xor_val);
const int8_t* wp = wptr + c0 * K0;
for (int k0 = 0; k0 < K0; k0++)
acc[k0] += ival * (int)wp[k0];
}
}
}
int8_t* optr = outptr + p * K0;
const int8_t* rptr = resptr ? resptr + p * K0 : nullptr;
for (int k0 = 0; k0 < K0; k0++) {
float val = (float)acc[k0] * multbuf[k0] + (float)out_zp;
if (rptr) {
int rval = inputIsU8 ? (int)(uint8_t)rptr[k0] : (int)rptr[k0];
val += (float)(rval - out_zp);
}
int ival = cvRound(val);
if (inputIsU8) {
ival = std::max(0, std::min(255, ival));
if (activLUT) ival = (int)(uint8_t)activLUT[ival];
((uint8_t*)optr)[k0] = (uint8_t)ival;
} else {
ival = std::max(-128, std::min(127, ival));
if (activLUT) ival = (int)activLUT[ival + 128];
optr[k0] = (int8_t)ival;
}
}
}
}
});
}
#endif // !CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace cv::dnn
modules/dnn/src/int8layers/conv2_int8_layer.cpp
+371
View File
@@ -0,0 +1,371 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "../net_impl.hpp"
#include "../layers/conv2_common.hpp"
#include "opencv2/core/hal/intrin.hpp"
#include "conv2_int8_kernels.simd.hpp"
#include "int8layers/conv2_int8_kernels.simd_declarations.hpp"
namespace cv {
namespace dnn {
static MatShape getWpackShapeInt8(const MatShape& wshape, int ngroups, int C0)
{
CV_Assert(wshape.dims >= 3);
int K = wshape[0], Cg = wshape[1];
int ksize = (int)(wshape.total()) / (K * Cg);
CV_Assert(K % ngroups == 0);
int Kg = K / ngroups, K0 = C0;
int Kblk = (Kg + K0 - 1) / K0;
int C1Max = 0;
for (int g = 0; g < ngroups; ++g) {
int c_start = g * Cg;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) / C0;
C1Max = std::max(C1Max, cblocks);
}
return MatShape({ngroups, Kblk, ksize, C1Max, C0 * K0}, DATA_LAYOUT_UNKNOWN);
}
static void repackConvWeightsInt8(const Mat& weights, Mat& Wpack, int ngroups, int C0_)
{
CV_Assert(weights.isContinuous());
CV_Assert(weights.type() == CV_8SC1);
CV_Assert(ngroups > 0);
CV_Assert((C0_ & (C0_ - 1)) == 0 && C0_ >= 4);
MatShape wshape = weights.shape();
CV_Assert(wshape.dims >= 3);
int K = wshape[0];
CV_Assert(K % ngroups == 0);
if (!Wpack.isContinuous())
Wpack.release();
MatShape wpackShape = getWpackShapeInt8(wshape, ngroups, C0_);
Wpack.create(wpackShape, CV_8SC1);
Wpack.setZero();
parallel_for_(Range(0, K), [&](const Range& range) {
int Cg = wshape[1], Kg = K / ngroups;
int ksize = wpackShape[2], C1Max = wpackShape[3];
int C0 = C0_, K0 = C0;
const int8_t* wdata = weights.ptr<int8_t>();
int8_t* Wpackdata = Wpack.ptr<int8_t>();
for (int k = range.start; k < range.end; ++k) {
int g = k / Kg;
int kin = k - g * Kg;
int kblk = kin / K0;
int k0 = kin & (K0 - 1);
int c_start = g * Cg;
int c00 = c_start & (C0 - 1);
for (int c = 0; c < Cg; ++c) {
int ch = c00 + c;
int c1 = ch / C0;
int c0 = ch & (C0 - 1);
const int8_t* wptr = wdata + ((k * Cg + c) * ksize);
int8_t* wpackptr = Wpackdata +
(((g * (int64_t)wpackShape[1] + kblk) * ksize * C1Max + c1) * C0 + c0) * K0 + k0;
for (int i = 0; i < ksize; ++i) {
wpackptr[i * (C1Max * C0 * K0)] = wptr[i];
}
}
}
});
}
static void repackWeightsForVNNI(const Mat& wpack, int ngroups, int Kg, int Cg,
const Mat& biasInt32,
Mat& wpackVNNI, Mat& biasVNNI)
{
constexpr int C0 = 8, K0 = 8;
MatShape ws = wpack.shape();
int Kblk = ws[1], ksize = ws[2], C1Max = ws[3];
wpackVNNI.create(ws, CV_8SC1);
const int8_t* src = wpack.ptr<int8_t>();
int8_t* dst = wpackVNNI.ptr<int8_t>();
int blockSize = C0 * K0;
int totalBlocks = (int)(ws.total() / blockSize);
parallel_for_(Range(0, totalBlocks), [&](const Range& range) {
for (int b = range.start; b < range.end; b++) {
const int8_t* s = src + (size_t)b * blockSize;
int8_t* d = dst + (size_t)b * blockSize;
for (int k = 0; k < K0; k++) {
d[k*4 + 0] = s[0*K0 + k];
d[k*4 + 1] = s[1*K0 + k];
d[k*4 + 2] = s[2*K0 + k];
d[k*4 + 3] = s[3*K0 + k];
d[32 + k*4 + 0] = s[4*K0 + k];
d[32 + k*4 + 1] = s[5*K0 + k];
d[32 + k*4 + 2] = s[6*K0 + k];
d[32 + k*4 + 3] = s[7*K0 + k];
}
}
});
int K = ngroups * Kg;
biasVNNI.create({K}, CV_32SC1);
const int32_t* bsrc = biasInt32.ptr<int32_t>();
int32_t* bdst = biasVNNI.ptr<int32_t>();
parallel_for_(Range(0, K), [&](const Range& range) {
for (int k = range.start; k < range.end; k++) {
int g = k / Kg;
int kin = k - g * Kg;
int kblk = kin / K0;
int k0 = kin & (K0 - 1);
int c_start = g * Cg;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) / C0;
const int8_t* wbase = src + (size_t)((g * Kblk + kblk) * ksize * C1Max) * C0 * K0;
int32_t wsum = 0;
for (int ki = 0; ki < ksize; ki++)
for (int c1 = 0; c1 < cblocks; c1++)
for (int c0 = 0; c0 < C0; c0++)
wsum += (int)wbase[((size_t)ki * C1Max + c1) * C0 * K0 + c0 * K0 + k0];
bdst[k] = bsrc[k] - 128 * wsum;
}
});
}
static void convInt8Block(const void* inp_, const void* residual_,
void* out_, const ConvState& cs,
const void* weights_,
const void* weightsVNNI_,
const int* bias, const int* biasVNNI_,
const float* multiplier,
int inp_zp, int out_zp,
const int8_t* activLUT,
bool inputIsU8)
{
#if CV_TRY_AVX2
if (cv::checkHardwareSupport(CV_CPU_AVX2)) {
opt_AVX2::convInt8Block(inp_, residual_, out_, cs, weights_,
weightsVNNI_, bias, biasVNNI_,
multiplier, inp_zp, out_zp, activLUT, inputIsU8);
return;
}
#endif
CV_CPU_CALL_BASELINE(convInt8Block, (inp_, residual_, out_, cs, weights_,
weightsVNNI_, bias, biasVNNI_,
multiplier, inp_zp, out_zp, activLUT, inputIsU8));
}
class Conv2Int8LayerImpl CV_FINAL : public Conv2Int8Layer
{
public:
Mat weights; // repacked int8 weights in block format
Mat weightsVNNI; // VNNI-transposed weights (pre-computed)
Mat biasInt32; // int32 fused bias
Mat biasVNNI; // int32 VNNI-adjusted bias (pre-computed)
Mat outMultiplier; // float32 per-channel output multiplier
MatShape wshape0; // original weight shape (K x Cg x kH x kW)
MatShape prevInpshape;
ConvState cs;
Mat activationLUT;
Ptr<ActivationLayerInt8> activ;
bool addResidual;
bool inputIsU8;
Conv2Int8LayerImpl(const LayerParams& params)
{
setParamsFrom(params);
auto_pad = getAutoPadding(params);
ceil_mode = params.get<bool>("ceil_mode", false);
strides = params.getVector<int>("stride");
dilations = params.getVector<int>("dilation");
pads = params.getVector<int>("pad");
ngroups = params.get<int>("group", 1);
input_sc = params.get<float>("input_scale", 1.f);
input_zp = params.get<int>("input_zeropoint", 0);
output_sc = params.get<float>("scales", 1.f);
output_zp = params.get<int>("zeropoints", 0);
per_channel = params.get<bool>("per_channel", true);
inputIsU8 = params.get<bool>("input_is_u8", false);
addResidual = false;
if (!blobs.empty()) {
CV_Assert(blobs.size() >= 3);
wshape0 = blobs[0].shape();
biasInt32 = blobs[1];
outMultiplier = blobs[2];
}
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs, const int,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
// Input can be CV_8SC1 or CV_8UC1; output matches input type
int outtype = !inputs.empty() ? inputs[0] : CV_8SC1;
outputs.assign(requiredOutputs, outtype);
internals.clear();
}
bool getMemoryShapes(const std::vector<MatShape>& inpshapes,
const int,
std::vector<MatShape>& outshapes,
std::vector<MatShape>& tempshapes) const CV_OVERRIDE
{
size_t ninputs = inpshapes.size();
CV_Assert(ninputs >= 1);
std::vector<int> emptyKernelShape;
outshapes.assign(1, convInferShape(inpshapes[0], wshape0, emptyKernelShape,
ngroups, strides, dilations,
pads, auto_pad, ceil_mode));
tempshapes.clear();
return true;
}
int getLayouts(const std::vector<DataLayout>& actualInputs,
std::vector<DataLayout>& desiredInputs,
const int requiredOutputs,
std::vector<DataLayout>& outputs) const CV_OVERRIDE
{
size_t ninputs = actualInputs.size();
CV_Assert(ninputs >= 1u && requiredOutputs == 1u);
desiredInputs = actualInputs;
desiredInputs[0] = DATA_LAYOUT_BLOCK;
for (size_t i = 1; i < ninputs; i++)
desiredInputs[i] = DATA_LAYOUT_UNKNOWN;
outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK);
return getNetImpl(this)->defaultC0;
}
void setWeightsInt8(const Mat& w_q, const Mat& bias, const Mat& multiplier, int C0)
{
CV_Assert(w_q.type() == CV_8SC1);
wshape0 = w_q.shape();
biasInt32 = bias;
outMultiplier = multiplier;
repackConvWeightsInt8(w_q, weights, ngroups, C0);
}
bool fuseAddResidual(Arg /*residual*/)
{
addResidual = true;
return true;
}
bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
{
Ptr<ActivationLayerInt8> activ_int8 = layer.dynamicCast<ActivationLayerInt8>();
if (!activ_int8.empty()) {
activ = activ_int8;
if (!activ_int8->blobs.empty())
activ_int8->blobs[0].convertTo(activationLUT, CV_8S);
return true;
}
return false;
}
void forward(InputArrayOfArrays input_arrs,
OutputArrayOfArrays output_arrs,
OutputArrayOfArrays) CV_OVERRIDE
{
int ninputs = (int)input_arrs.total();
CV_Assert(ninputs >= 1);
const Mat& inp = input_arrs.getMat(0);
MatShape inpshape = inp.shape();
CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(inp.type() == CV_8SC1 || inp.type() == CV_8UC1);
int C0 = inpshape.back();
if (weights.empty() && !blobs.empty()) {
repackConvWeightsInt8(blobs[0], weights, ngroups, C0);
}
if (weightsVNNI.empty() && !weights.empty()) {
int K = wshape0[0];
int Cg = wshape0[1];
int Kg = K / ngroups;
repackWeightsForVNNI(weights, ngroups, Kg, Cg,
biasInt32, weightsVNNI, biasVNNI);
}
Mat residual;
const void* resptr = nullptr;
if (addResidual) {
residual = input_arrs.getMat(ninputs - 1);
resptr = residual.data;
ninputs--;
}
std::vector<int> emptyKernelShape;
MatShape outshape = convInferShape(inpshape, wshape0, emptyKernelShape,
ngroups, strides, dilations,
pads, auto_pad, ceil_mode);
int outtype = inp.type();
int outkind = output_arrs.kind();
Mat out;
if (outkind == _InputArray::STD_VECTOR_MAT) {
std::vector<Mat>& outs = output_arrs.getMatVecRef();
outs.resize(1);
outs[0].fit(outshape, outtype);
out = outs[0];
} else {
std::vector<UMat>& outs = output_arrs.getUMatVecRef();
outs.resize(1);
outs[0].fit(outshape, outtype);
out.fit(outshape, outtype);
}
if (inpshape != prevInpshape) {
cs.initConv(inpshape, wshape0, outshape, ngroups,
strides, dilations, pads, auto_pad, ceil_mode,
FAST_ACTIV_NONE, {});
prevInpshape = inpshape;
}
const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr<int8_t>() : nullptr;
convInt8Block(inp.data, resptr, out.data, cs,
weights.data,
weightsVNNI.empty() ? nullptr : weightsVNNI.data,
biasInt32.ptr<int>(),
biasVNNI.empty() ? nullptr : biasVNNI.ptr<int>(),
outMultiplier.ptr<float>(),
input_zp, output_zp,
lutptr, inputIsU8);
if (outkind == _InputArray::STD_VECTOR_UMAT) {
std::vector<UMat>& outs = output_arrs.getUMatVecRef();
out.copyTo(outs[0]);
}
}
};
Ptr<Conv2Int8Layer> Conv2Int8Layer::create(const LayerParams& params)
{
return Ptr<Conv2Int8Layer>(new Conv2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/int8layers/eltwise2_int8_layer.cpp
+413
View File
@@ -0,0 +1,413 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "../net_impl.hpp"
#include "opencv2/core/hal/intrin.hpp"
#if CV_AVX2
#include <immintrin.h>
#endif
#if CV_NEON
#include <arm_neon.h>
#endif
namespace cv {
namespace dnn {
class Eltwise2Int8LayerImpl CV_FINAL : public Eltwise2Int8Layer
{
public:
std::vector<float> coeffs;
float offset;
bool withRelu;
Mat activationLUT;
Ptr<ActivationLayerInt8> activ;
Eltwise2Int8LayerImpl(const LayerParams& params)
{
setParamsFrom(params);
output_zp = params.get<int>("zeropoints", 0);
output_sc = params.get<float>("scales", 1.0f);
withRelu = params.get<bool>("with_relu", false);
if (params.has("input_scales"))
{
DictValue sc = params.get("input_scales");
int n = sc.size();
scales.resize(n);
for (int i = 0; i < n; i++)
scales[i] = sc.get<float>(i);
}
if (params.has("input_zeropoints"))
{
DictValue zp = params.get("input_zeropoints");
int n = zp.size();
zeropoints.resize(n);
for (int i = 0; i < n; i++)
zeropoints[i] = zp.get<int>(i);
}
offset = 0.f;
}
void ensureCoeffs()
{
if (!coeffs.empty() || scales.empty())
return;
CV_CheckEQ(scales.size(), zeropoints.size(),
"Eltwise2Int8: scales and zeropoints sizes must match");
CV_Assert(output_sc > 0.f);
coeffs.resize(scales.size());
offset = (float)output_zp;
for (size_t i = 0; i < scales.size(); i++)
{
coeffs[i] = scales[i] / output_sc;
offset -= coeffs[i] * zeropoints[i];
}
}
bool getMemoryShapes(const std::vector<MatShape>& inputs,
const int requiredOutputs,
std::vector<MatShape>& outputs,
std::vector<MatShape>& internals) const CV_OVERRIDE
{
CV_Assert(inputs.size() >= 2);
outputs.assign(1, inputs[0]);
internals.clear();
return true;
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_Assert(!inputs.empty());
outputs.assign(requiredOutputs, inputs[0]);
internals.clear();
}
int getLayouts(const std::vector<DataLayout>& actualInputs,
std::vector<DataLayout>& desiredInputs,
const int requiredOutputs,
std::vector<DataLayout>& outputs) const CV_OVERRIDE
{
auto* netimpl_ = getNetImpl(this);
DataLayout defaultLayout = netimpl_->originalLayout;
size_t ninputs = actualInputs.size(), nblockInputs = 0;
CV_Assert(ninputs >= 1u);
for (size_t i = 0; i < ninputs; i++)
nblockInputs += actualInputs[i] == DATA_LAYOUT_BLOCK;
desiredInputs = actualInputs;
if (nblockInputs == ninputs) {
outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK);
} else {
if (nblockInputs < ninputs) {
for (size_t i = 0; i < ninputs; i++) {
DataLayout layout = actualInputs[i];
desiredInputs[i] = layout == DATA_LAYOUT_BLOCK ? defaultLayout : layout;
}
}
outputs.assign(requiredOutputs, DATA_LAYOUT_UNKNOWN);
}
return outputs[0] == DATA_LAYOUT_BLOCK ? netimpl_->defaultC0 : 0;
}
void forward(InputArrayOfArrays input_arrs,
OutputArrayOfArrays output_arrs,
OutputArrayOfArrays) CV_OVERRIDE
{
ensureCoeffs();
int ninputs = (int)input_arrs.total();
CV_Assert(ninputs >= 2 && coeffs.size() == (size_t)ninputs);
const Mat& inp0 = input_arrs.getMat(0);
MatShape outshape = inp0.shape();
int outtype = inp0.type();
CV_Assert(outtype == CV_8SC1 || outtype == CV_8UC1);
const bool isU8 = (outtype == CV_8UC1);
int outkind = output_arrs.kind();
std::vector<Mat>* outs = nullptr;
std::vector<UMat>* uouts = nullptr;
Mat out;
if (outkind == _InputArray::STD_VECTOR_MAT) {
outs = &output_arrs.getMatVecRef();
outs->resize(1);
outs->at(0).fit(outshape, outtype);
out = outs->at(0);
} else {
uouts = &output_arrs.getUMatVecRef();
uouts->resize(1);
uouts->at(0).fit(outshape, outtype);
out.fit(outshape, outtype);
}
const size_t total_elems = inp0.total();
const float* cptr = coeffs.data();
const float off = offset;
const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr<int8_t>() : nullptr;
const float c0 = cptr[0];
const float c1 = cptr[1];
const int out_min = isU8 ? (withRelu ? output_zp : 0) : (withRelu ? output_zp : -128);
const int out_max = isU8 ? 255 : 127;
const double nstripes = (double)std::max(1, getNumThreads() * 4);
if (isU8) {
std::vector<const uint8_t*> inptrs(ninputs);
for (int k = 0; k < ninputs; k++)
inptrs[k] = input_arrs.getMat(k).ptr<uint8_t>();
const uint8_t* p0 = inptrs[0];
const uint8_t* p1 = inptrs[1];
uint8_t* outptr = out.ptr<uint8_t>();
parallel_for_(Range(0, (int)total_elems), [&](const Range& r) {
int i = r.start;
#if CV_AVX2
if (!lutptr && ninputs == 2) {
__m256 vc0 = _mm256_set1_ps(c0);
__m256 vc1 = _mm256_set1_ps(c1);
__m256 voff = _mm256_set1_ps(off);
__m256i vmin = _mm256_set1_epi32(out_min);
__m256i vmax = _mm256_set1_epi32(out_max);
for (; i <= r.end - 8; i += 8) {
__m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i));
__m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i));
__m256i a32 = _mm256_cvtepu8_epi32(a8);
__m256i b32 = _mm256_cvtepu8_epi32(b8);
__m256 af = _mm256_cvtepi32_ps(a32);
__m256 bf = _mm256_cvtepi32_ps(b32);
__m256 val = _mm256_fmadd_ps(vc0, af, _mm256_fmadd_ps(vc1, bf, voff));
__m256i ival = _mm256_cvtps_epi32(val);
ival = _mm256_max_epi32(ival, vmin);
ival = _mm256_min_epi32(ival, vmax);
__m128i lo = _mm256_castsi256_si128(ival);
__m128i hi = _mm256_extracti128_si256(ival, 1);
__m128i packed16 = _mm_packs_epi32(lo, hi);
__m128i packed8 = _mm_packus_epi16(packed16, packed16);
_mm_storel_epi64((__m128i*)(outptr + i), packed8);
}
}
#elif CV_NEON
if (ninputs == 2) {
float32x4_t vc0 = vdupq_n_f32(c0);
float32x4_t vc1 = vdupq_n_f32(c1);
float32x4_t voff = vdupq_n_f32(off);
int32x4_t vmin = vdupq_n_s32(out_min);
int32x4_t vmax = vdupq_n_s32(out_max);
for (; i <= r.end - 8; i += 8) {
uint8x8_t a8 = vld1_u8(p0 + i);
uint8x8_t b8 = vld1_u8(p1 + i);
uint16x8_t a16 = vmovl_u8(a8);
uint16x8_t b16 = vmovl_u8(b8);
int32x4_t a32_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(a16)));
int32x4_t b32_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(b16)));
int32x4_t a32_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(a16)));
int32x4_t b32_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(b16)));
float32x4_t val_lo = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_lo))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_lo)));
float32x4_t val_hi = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_hi))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_hi)));
#if CV_NEON_AARCH64
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_lo), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_hi), vmin), vmax);
#else
float32x4_t half = vdupq_n_f32(0.5f);
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_lo,
vbslq_f32(vcgeq_f32(val_lo, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_hi,
vbslq_f32(vcgeq_f32(val_hi, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
#endif
int16x8_t p16 = vcombine_s16(vqmovn_s32(ival_lo), vqmovn_s32(ival_hi));
uint8x8_t p8u = vqmovun_s16(p16);
if (lutptr) {
uint8_t tmp[8];
vst1_u8(tmp, p8u);
for (int k = 0; k < 8; k++)
outptr[i + k] = (uint8_t)lutptr[tmp[k]];
} else {
vst1_u8(outptr + i, p8u);
}
}
}
#endif
for (; i < r.end; i++)
{
float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off;
for (int k = 2; k < ninputs; k++)
val += cptr[k] * (float)inptrs[k][i];
int ival = cvRound(val);
ival = std::max(out_min, std::min(out_max, ival));
if (lutptr)
ival = (int)(uint8_t)lutptr[ival];
outptr[i] = (uint8_t)ival;
}
}, nstripes);
} else {
std::vector<const int8_t*> inptrs(ninputs);
for (int k = 0; k < ninputs; k++)
inptrs[k] = input_arrs.getMat(k).ptr<int8_t>();
const int8_t* p0 = inptrs[0];
const int8_t* p1 = inptrs[1];
int8_t* outptr = out.ptr<int8_t>();
parallel_for_(Range(0, (int)total_elems), [&](const Range& r) {
int i = r.start;
#if CV_AVX2
if (!lutptr && ninputs == 2) {
__m256 vc0 = _mm256_set1_ps(c0);
__m256 vc1 = _mm256_set1_ps(c1);
__m256 voff = _mm256_set1_ps(off);
__m256i vmin = _mm256_set1_epi32(out_min);
__m256i vmax = _mm256_set1_epi32(out_max);
for (; i <= r.end - 8; i += 8) {
__m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i));
__m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i));
__m256i a32 = _mm256_cvtepi8_epi32(a8);
__m256i b32 = _mm256_cvtepi8_epi32(b8);
__m256 af = _mm256_cvtepi32_ps(a32);
__m256 bf = _mm256_cvtepi32_ps(b32);
__m256 val = _mm256_fmadd_ps(vc0, af, _mm256_fmadd_ps(vc1, bf, voff));
__m256i ival = _mm256_cvtps_epi32(val);
ival = _mm256_max_epi32(ival, vmin);
ival = _mm256_min_epi32(ival, vmax);
__m128i lo = _mm256_castsi256_si128(ival);
__m128i hi = _mm256_extracti128_si256(ival, 1);
__m128i packed16 = _mm_packs_epi32(lo, hi);
__m128i packed8 = _mm_packs_epi16(packed16, packed16);
_mm_storel_epi64((__m128i*)(outptr + i), packed8);
}
}
#elif CV_NEON
if (ninputs == 2) {
float32x4_t vc0 = vdupq_n_f32(c0);
float32x4_t vc1 = vdupq_n_f32(c1);
float32x4_t voff = vdupq_n_f32(off);
int32x4_t vmin = vdupq_n_s32(out_min);
int32x4_t vmax = vdupq_n_s32(out_max);
for (; i <= r.end - 8; i += 8) {
int8x8_t a8 = vld1_s8(p0 + i);
int8x8_t b8 = vld1_s8(p1 + i);
int16x8_t a16 = vmovl_s8(a8);
int16x8_t b16 = vmovl_s8(b8);
int32x4_t a32_lo = vmovl_s16(vget_low_s16(a16));
int32x4_t b32_lo = vmovl_s16(vget_low_s16(b16));
int32x4_t a32_hi = vmovl_s16(vget_high_s16(a16));
int32x4_t b32_hi = vmovl_s16(vget_high_s16(b16));
float32x4_t val_lo = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_lo))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_lo)));
float32x4_t val_hi = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_hi))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_hi)));
#if CV_NEON_AARCH64
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_lo), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_hi), vmin), vmax);
#else
float32x4_t half = vdupq_n_f32(0.5f);
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_lo,
vbslq_f32(vcgeq_f32(val_lo, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_hi,
vbslq_f32(vcgeq_f32(val_hi, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
#endif
int16x8_t p16 = vcombine_s16(vqmovn_s32(ival_lo), vqmovn_s32(ival_hi));
int8x8_t p8 = vqmovn_s16(p16);
if (lutptr) {
int8_t tmp[8];
vst1_s8(tmp, p8);
for (int k = 0; k < 8; k++)
outptr[i + k] = (int8_t)lutptr[(int)tmp[k] + 128];
} else {
vst1_s8(outptr + i, p8);
}
}
}
#endif
for (; i < r.end; i++)
{
float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off;
for (int k = 2; k < ninputs; k++)
val += cptr[k] * (float)inptrs[k][i];
int ival = cvRound(val);
ival = std::max(out_min, std::min(out_max, ival));
if (lutptr)
ival = (int)lutptr[ival + 128];
outptr[i] = (int8_t)ival;
}
}, nstripes);
}
if (uouts) {
out.copyTo(uouts->at(0));
}
}
bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
{
Ptr<ActivationLayerInt8> activ_int8 = layer.dynamicCast<ActivationLayerInt8>();
if (!activ_int8.empty())
{
activ = activ_int8;
if (!activ_int8->blobs.empty())
activationLUT = activ_int8->blobs[0];
return true;
}
return false;
}
};
Ptr<Eltwise2Int8Layer> Eltwise2Int8Layer::create(const LayerParams& params)
{
return Ptr<Eltwise2Int8Layer>(new Eltwise2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/int8layers/fully_connected_layer.cpp
+22 -3
View File
@@ -28,6 +28,7 @@ public:
output_sc = params.get<float>("scales", 1.0f);
axis = params.get<int>("axis", 1);
per_channel = params.get<bool>("per_channel", true);
output_type = CV_8S; // default, may be overridden by fusion code
if (blobs.size() == 3)
{
@@ -77,6 +78,16 @@ public:
return false;
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
outputs.assign(requiredOutputs, output_type);
internals.clear();
}
virtual bool supportBackend(int backendId) CV_OVERRIDE
{
if (backendId == DNN_BACKEND_TIMVX && haveTimVX())
@@ -383,14 +394,22 @@ public:
int axisCan = normalize_axis(axis, input[0].dims);
int outerSize = input[0].total(0, axisCan);
Mat srcMat = input[0].reshape(1, outerSize);
Mat srcMat0 = input[0].reshape(1, outerSize);
Mat srcMat;
if (srcMat0.type() == CV_8U) {
// Convert uint8 to int8 by subtracting 128.
// Zero-point was adjusted at fusion time to compensate.
srcMat0.convertTo(srcMat, CV_8S, 1, -128);
} else {
srcMat = srcMat0;
}
Mat dstMat = output[0].reshape(1, outerSize);
Mat dstMatInt32= Mat(shape(dstMat), CV_32S);
const int nstripes = getNumThreads();
const int nstripes = outerSize <= 4 ? 1 : getNumThreads();
FullyConnected::run(srcMat, weightsMat, biasMat, outputMultiplier, activationLUT, dstMatInt32, activ.get(), nstripes, output_zp);
dstMatInt32.convertTo(dstMat, CV_8S);
dstMatInt32.convertTo(dstMat, output_type);
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
modules/dnn/src/int8layers/pool2_int8_layer.cpp
+381
View File
@@ -0,0 +1,381 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners
#include "../precomp.hpp"
#include "../net_impl.hpp"
#include "../layers/conv2_common.hpp"
#include "opencv2/core/hal/intrin.hpp"
namespace cv {
namespace dnn {
template <typename T>
static void maxPoolImpl(const void* inp_, void* out_, const ConvState& cs)
{
constexpr int MAX_POOL_DIMS = ConvState::MAX_CONV_DIMS;
int NC1 = cs.inpshape[0] * cs.inpshape[1];
CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.inpshape.dims == cs.outshape.dims);
parallel_for_(Range(0, NC1), [&](const Range& r) {
CV_Assert(cs.nspatialdims <= MAX_POOL_DIMS && MAX_POOL_DIMS == 3);
int sdims = cs.nspatialdims;
int nc0 = r.start, nc1 = r.end;
int C0 = cs.inpshape.back();
int Di = sdims > 2 ? cs.inpshape[sdims - 1] : 1;
int Hi = sdims > 1 ? cs.inpshape[sdims] : 1;
int Wi = cs.inpshape[sdims + 1];
int D = sdims > 2 ? cs.outshape[sdims - 1] : 1;
int H = sdims > 1 ? cs.outshape[sdims] : 1;
int W = cs.outshape[sdims + 1];
int iplanesize = Di * Hi * Wi * C0;
int planesize = D * H * W * C0;
int SZ = cs.strides[0], SY = cs.strides[1], SX = cs.strides[2];
int padZ0 = cs.pads[0], padY0 = cs.pads[1], padX0 = cs.pads[2];
int inner_z0 = cs.inner[0], inner_z1 = cs.inner[MAX_POOL_DIMS];
int inner_y0 = cs.inner[1], inner_y1 = cs.inner[MAX_POOL_DIMS + 1];
int inner_x0 = cs.inner[2], inner_x1 = cs.inner[MAX_POOL_DIMS + 2];
int ksize = (int)cs.ofstab.size();
const int* zyxtab = cs.coordtab.data();
const int* ofstab = cs.ofstab.data();
const T* inp = (const T*)inp_ + nc0 * iplanesize;
T* out = (T*)out_ + nc0 * planesize;
const T INITVAL = std::numeric_limits<T>::min();
for (int nc = nc0; nc < nc1; nc++, inp += iplanesize) {
for (int z0 = 0; z0 < D; z0++) {
int zi_ = z0 * SZ - padZ0;
for (int y0 = 0; y0 < H; y0++, out += W * C0) {
int x0 = 0;
int x1 = z0 >= inner_z0 && z0 < inner_z1 &&
y0 >= inner_y0 && y0 < inner_y1 ? inner_x0 : W;
int yi_ = y0 * SY - padY0;
for (;;) {
for (; x0 < x1; x0++) {
int xi_ = x0 * SX - padX0;
for (int c = 0; c < C0; c++)
out[x0 * C0 + c] = INITVAL;
for (int k = 0; k < ksize; k++) {
int zi = zi_ + zyxtab[k * MAX_POOL_DIMS];
int yi = yi_ + zyxtab[k * MAX_POOL_DIMS + 1];
int xi = xi_ + zyxtab[k * MAX_POOL_DIMS + 2];
if ((unsigned)zi >= (unsigned)Di ||
(unsigned)yi >= (unsigned)Hi ||
(unsigned)xi >= (unsigned)Wi)
continue;
const T* inptr = inp + ((zi * Hi + yi) * Wi + xi) * C0;
for (int c = 0; c < C0; c++)
out[x0 * C0 + c] = std::max(out[x0 * C0 + c], inptr[c]);
}
}
if (x0 == W)
break;
x1 = inner_x1;
for (; x0 < x1; x0++) {
int xi_ = x0 * SX - padX0;
const T* inp_xi = inp + ((Hi * zi_ + yi_) * Wi + xi_) * C0;
for (int c = 0; c < C0; c++) {
T s = inp_xi[ofstab[0] + c];
for (int k = 1; k < ksize; k++)
s = std::max(s, inp_xi[ofstab[k] + c]);
out[x0 * C0 + c] = s;
}
}
x1 = W;
}
}
}
}
});
}
static void maxPoolInt8(const void* inp_, void* out_, const ConvState& cs, bool isU8)
{
if (isU8)
maxPoolImpl<uint8_t>(inp_, out_, cs);
else
maxPoolImpl<int8_t>(inp_, out_, cs);
}
static void avgPoolInt8(const void* inp_, void* out_, const ConvState& cs,
float inp_sc, int inp_zp, float out_sc, int out_zp,
bool count_include_pad_, bool isU8)
{
constexpr int MAX_POOL_DIMS = ConvState::MAX_CONV_DIMS;
int NC1 = cs.inpshape[0] * cs.inpshape[1];
CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.inpshape.dims == cs.outshape.dims);
parallel_for_(Range(0, NC1), [&](const Range& r) {
CV_Assert(cs.nspatialdims <= MAX_POOL_DIMS && MAX_POOL_DIMS == 3);
bool count_include_pad = count_include_pad_;
int sdims = cs.nspatialdims;
int nc0 = r.start, nc1 = r.end;
int C0 = cs.inpshape.back();
int Di = sdims > 2 ? cs.inpshape[sdims - 1] : 1;
int Hi = sdims > 1 ? cs.inpshape[sdims] : 1;
int Wi = cs.inpshape[sdims + 1];
int D = sdims > 2 ? cs.outshape[sdims - 1] : 1;
int H = sdims > 1 ? cs.outshape[sdims] : 1;
int W = cs.outshape[sdims + 1];
int iplanesize = Di * Hi * Wi * C0;
int planesize = D * H * W * C0;
int SZ = cs.strides[0], SY = cs.strides[1], SX = cs.strides[2];
int padZ0 = cs.pads[0], padY0 = cs.pads[1], padX0 = cs.pads[2];
int ksize_total = (int)cs.ofstab.size();
const int* zyxtab = cs.coordtab.data();
const uint8_t* inp = (const uint8_t*)inp_ + nc0 * iplanesize;
uint8_t* out = (uint8_t*)out_ + nc0 * planesize;
float scale_ratio = inp_sc / out_sc;
std::vector<int> accum(C0);
for (int nc = nc0; nc < nc1; nc++, inp += iplanesize) {
for (int z0 = 0; z0 < D; z0++) {
int zi_ = z0 * SZ - padZ0;
for (int y0 = 0; y0 < H; y0++, out += W * C0) {
int yi_ = y0 * SY - padY0;
for (int x0 = 0; x0 < W; x0++) {
int xi_ = x0 * SX - padX0;
for (int c = 0; c < C0; c++)
accum[c] = 0;
int count = 0;
for (int k = 0; k < ksize_total; k++) {
int zi = zi_ + zyxtab[k * MAX_POOL_DIMS];
int yi = yi_ + zyxtab[k * MAX_POOL_DIMS + 1];
int xi = xi_ + zyxtab[k * MAX_POOL_DIMS + 2];
if ((unsigned)zi >= (unsigned)Di ||
(unsigned)yi >= (unsigned)Hi ||
(unsigned)xi >= (unsigned)Wi) {
if (count_include_pad)
count++;
continue;
}
count++;
const uint8_t* inptr = inp + ((zi * Hi + yi) * Wi + xi) * C0;
for (int c = 0; c < C0; c++) {
int v = isU8 ? (int)inptr[c]
: (int)(int8_t)inptr[c];
accum[c] += v - inp_zp;
}
}
if (count == 0) count = 1;
float inv_count = 1.f / count;
for (int c = 0; c < C0; c++) {
float val = (float)accum[c] * inv_count * scale_ratio + (float)out_zp;
int ival = cvRound(val);
if (isU8)
out[x0 * C0 + c] = (uint8_t)std::max(0, std::min(255, ival));
else
out[x0 * C0 + c] = (uint8_t)(int8_t)std::max(-128, std::min(127, ival));
}
}
}
}
}
});
}
static void globalAvgPoolInt8(const void* inp_, void* out_,
int N, int C1, int spatialSize, int C0,
float inp_sc, int inp_zp, float out_sc, int out_zp,
bool isU8)
{
int NC1 = N * C1;
float scale_ratio = inp_sc / out_sc;
parallel_for_(Range(0, NC1), [&](const Range& r) {
for (int nc = r.start; nc < r.end; nc++) {
const uint8_t* inptr = (const uint8_t*)inp_ + (size_t)nc * spatialSize * C0;
uint8_t* outptr = (uint8_t*)out_ + (size_t)nc * C0;
for (int c = 0; c < C0; c++) {
int sum = 0;
for (int s = 0; s < spatialSize; s++) {
int v = isU8 ? (int)inptr[s * C0 + c]
: (int)(int8_t)inptr[s * C0 + c];
sum += v - inp_zp;
}
float val = (float)sum / spatialSize * scale_ratio + (float)out_zp;
int ival = cvRound(val);
if (isU8)
outptr[c] = (uint8_t)std::max(0, std::min(255, ival));
else
outptr[c] = (uint8_t)(int8_t)std::max(-128, std::min(127, ival));
}
}
});
}
class Pool2Int8LayerImpl CV_FINAL : public Pool2Int8Layer
{
public:
bool count_include_pad;
ConvState cs;
MatShape prevInpshape;
Pool2Int8LayerImpl(const LayerParams& params)
{
setParamsFrom(params);
auto_pad = getAutoPadding(params);
kernel_shape = params.getVector<int>("kernel_size");
strides = params.getVector<int>("stride");
dilations = params.getVector<int>("dilation");
pads = params.getVector<int>("pad");
ceil_mode = params.get<bool>("ceil_mode", false);
is_global_pooling = params.get<bool>("global_pooling", false);
is_max_pool = params.get<bool>("is_max_pool", true);
count_include_pad = params.get<bool>("count_include_pad", false);
input_sc = params.get<float>("input_scale", 1.f);
input_zp = params.get<int>("input_zeropoint", 0);
output_sc = params.get<float>("scales", 1.f);
output_zp = params.get<int>("zeropoints", 0);
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs, const int,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_Assert(!inputs.empty());
outputs.assign(requiredOutputs, inputs[0]);
internals.clear();
}
bool getMemoryShapes(const std::vector<MatShape>& inpshapes,
const int,
std::vector<MatShape>& outshapes,
std::vector<MatShape>& tempshapes) const CV_OVERRIDE
{
CV_Assert(inpshapes.size() == 1);
if (is_global_pooling) {
MatShape outshape = inpshapes[0];
for (int d = 2; d < outshape.dims - 1; d++)
outshape[d] = 1;
outshapes.assign(1, outshape);
} else {
outshapes.assign(1, convInferShape(inpshapes[0], MatShape(),
kernel_shape, 0, strides, dilations,
pads, auto_pad, ceil_mode));
}
tempshapes.clear();
return true;
}
int getLayouts(const std::vector<DataLayout>& actualInputs,
std::vector<DataLayout>& desiredInputs,
const int requiredOutputs,
std::vector<DataLayout>& outputs) const CV_OVERRIDE
{
CV_Assert(actualInputs.size() == 1u);
desiredInputs.assign(1, DATA_LAYOUT_BLOCK);
outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK);
return getNetImpl(this)->defaultC0;
}
void forward(InputArrayOfArrays inputs_arr,
OutputArrayOfArrays outputs_arr,
OutputArrayOfArrays) CV_OVERRIDE
{
size_t ninputs = inputs_arr.total();
CV_Assert(ninputs == 1);
const Mat& inp = inputs_arr.getMat(0);
int inptype = inp.type();
MatShape inpshape = inp.shape();
CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(inptype == CV_8SC1 || inptype == CV_8UC1);
MatShape outshape;
if (is_global_pooling) {
outshape = inpshape;
for (int d = 2; d < outshape.dims - 1; d++)
outshape[d] = 1;
} else {
outshape = convInferShape(inpshape, MatShape(),
kernel_shape, 0, strides, dilations,
pads, auto_pad, ceil_mode);
}
int outkind = outputs_arr.kind();
Mat out;
if (outkind == _InputArray::STD_VECTOR_MAT) {
std::vector<Mat>& outs = outputs_arr.getMatVecRef();
outs.resize(1);
outs[0].fit(outshape, inptype);
out = outs[0];
} else {
std::vector<UMat>& outs = outputs_arr.getUMatVecRef();
outs.resize(1);
outs[0].fit(outshape, inptype);
out.fit(outshape, inptype);
}
if (is_global_pooling) {
int N = inpshape[0];
int C1 = inpshape[1];
int C0 = inpshape.back();
int spatialSize = 1;
for (int d = 2; d < inpshape.dims - 1; d++)
spatialSize *= inpshape[d];
globalAvgPoolInt8(inp.data, out.data, N, C1, spatialSize, C0,
input_sc, input_zp, output_sc, output_zp,
inptype == CV_8UC1);
} else {
ConvState cs_local;
cs_local.initPooling(inpshape, outshape, kernel_shape, strides,
dilations, pads, auto_pad, ceil_mode);
if (is_max_pool) {
maxPoolInt8(inp.data, out.data, cs_local, inptype == CV_8UC1);
} else {
avgPoolInt8(inp.data, out.data, cs_local,
input_sc, input_zp, output_sc, output_zp,
count_include_pad, inptype == CV_8UC1);
}
}
if (outkind == _InputArray::STD_VECTOR_UMAT) {
std::vector<UMat>& outs = outputs_arr.getUMatVecRef();
out.copyTo(outs[0]);
}
}
};
Ptr<Pool2Int8Layer> Pool2Int8Layer::create(const LayerParams& params)
{
return Ptr<Pool2Int8Layer>(new Pool2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/layers/dequantizelinear_layer.cpp
+39 -30
View File
@@ -1,37 +1,54 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "../net_impl.hpp"
#if defined(__x86_64__) || defined(_M_X64)
#include <immintrin.h>
#endif
namespace cv
{
namespace dnn
{
#if CV_SIMD || CV_SIMD_SCALABLE
static void dequantizeLinearChunk_u8_f32(const uint8_t* src, float* dst,
float scale, int zp, int64_t len)
/*
DequantizeLinear layer, as defined in ONNX specification:
https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html
Opset's 10 to 25 are covered.
*/
#if defined(__x86_64__) || defined(_M_X64)
#if defined(__GNUC__) || defined(__clang__)
__attribute__((target("avx2")))
#endif
static void dequantizeLinearChunk_u8_f32_avx2(const uint8_t* src, float* dst,
float scale, int zp, int64_t len)
{
const int vlanes = VTraits<v_float32>::vlanes();
v_float32 vscale = vx_setall_f32(scale);
v_int32 vzp = vx_setall_s32(zp);
__m256 vscale = _mm256_set1_ps(scale);
__m256i vzp = _mm256_set1_epi32(zp);
int64_t j = 0;
for (; j <= len - vlanes; j += vlanes) {
v_int32 vi = v_reinterpret_as_s32(vx_load_expand_q(src + j));
vi = v_sub(vi, vzp);
v_float32 vf = v_mul(v_cvt_f32(vi), vscale);
v_store(dst + j, vf);
for (; j <= len - 8; j += 8) {
__m128i raw = _mm_loadl_epi64((__m128i*)(src + j));
__m256i vi = _mm256_cvtepu8_epi32(raw);
vi = _mm256_sub_epi32(vi, vzp);
__m256 vf = _mm256_cvtepi32_ps(vi);
vf = _mm256_mul_ps(vf, vscale);
_mm256_storeu_ps(dst + j, vf);
}
for (; j < len; j++)
dst[j] = (float)(src[j] - zp) * scale;
}
static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out,
float scale, int zp,
int64_t total)
static void dequantizeLinearFast_u8_f32_avx2(const uint8_t* inp, float* out,
float scale, int zp,
int64_t total)
{
const int64_t block = 1024;
int64_t nblocks = (total + block - 1) / block;
@@ -40,20 +57,12 @@ static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out,
for (int i = r.start; i < r.end; i++) {
int64_t ofs = i * block;
int64_t len = std::min(block, total - ofs);
dequantizeLinearChunk_u8_f32(inp + ofs, out + ofs, scale, zp, len);
dequantizeLinearChunk_u8_f32_avx2(inp + ofs, out + ofs, scale, zp, len);
}
});
}
#endif
/*
DequantizeLinear layer, as defined in ONNX specification:
https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html
Opset's 10 to 23 are covered.
*/
template <typename _InpTp, typename _ScaleTp, typename _OutTp>
static void dequantizeLinear(const _InpTp* inp_, const _ScaleTp* scale_,
const _InpTp* zp_, _OutTp* out_,
@@ -91,7 +100,6 @@ static void dequantizeLinear(const _InpTp* inp_, const _ScaleTp* scale_,
const _ScaleTp* sc = scale_ + scale_ofs;
_OutTp* out = out_ + block_ofs;
// [TODO] vectorize using intrinsics
if (slice_size > 1) {
for (int k = 0; k < delta; k++, inp += slice_size, out += slice_size,
sc += scale_step, zp += zp_step) {
@@ -201,15 +209,16 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp,
}
}
// Fast path: per-tensor dequantization uint8→float with universal intrinsics
#if CV_SIMD || CV_SIMD_SCALABLE
if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F) {
// Fast path: per-tensor dequantization uint8→float with AVX2 + proper parallelism
#if defined(__x86_64__) || defined(_M_X64)
if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F
&& checkHardwareSupport(CV_CPU_AVX2)) {
float sc = reinterpret_cast<const float*>(scale.data)[0];
int zpval = zp.empty() ? 0 : (int)reinterpret_cast<const uint8_t*>(zp.data)[0];
int64_t total = nslices * slice_size;
dequantizeLinearFast_u8_f32(reinterpret_cast<const uint8_t*>(inp.data),
reinterpret_cast<float*>(out.data),
sc, zpval, total);
dequantizeLinearFast_u8_f32_avx2(reinterpret_cast<const uint8_t*>(inp.data),
reinterpret_cast<float*>(out.data),
sc, zpval, total);
return;
}
#endif
modules/dnn/test/test_model.cpp
+57
View File
@@ -921,4 +921,61 @@ TEST_P(Reproducibility_ResNet50_ONNX, Accuracy)
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_ONNX,
testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)));
typedef testing::TestWithParam<Target> Reproducibility_ResNet50_QDQ_ONNX;
TEST_P(Reproducibility_ResNet50_QDQ_ONNX, Accuracy)
{
Target targetId = GetParam();
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16);
std::string modelname = _tf("onnx/models/resnet50-v1-12-qdq.onnx", false);
Net net = readNetFromONNX(modelname);
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId);
if (targetId == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
std::string imgname = _tf("sqcat.png");
Mat image = imread(imgname);
Mat input = blobFromImage(image, 0.017, Size(224,224),
Scalar(103.939, 116.779, 123.68),
false, true, CV_32F);
ASSERT_TRUE(!input.empty());
Mat out;
double min_t = 0;
const int niters =
#ifdef _DEBUG
1;
#else
30;
#endif
for (int i = 0; i < niters; i++) {
double t = (double)getTickCount();
net.setInput(input);
out = net.forward();
t = (double)getTickCount() - t;
min_t = i == 0 ? t : std::min(min_t, t);
}
printf("run time = %.2fms\n", min_t*1000./getTickFrequency());
const int K = 5;
std::vector<std::pair<int, float> > res;
topK(out, res, K);
ASSERT_EQ(int(res.size()), K);
std::vector<std::pair<int, float> > ref = {{285, 10.44}, {287, 10.13}, {283, 8.89}, {278, 8.43}};
const float eps = 0.5f;
for (int i = 0; i < (int)ref.size(); i++) {
EXPECT_EQ(ref[i].first, res[i].first);
EXPECT_NEAR(ref[i].second, res[i].second, eps);
}
}
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_QDQ_ONNX,
testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)));
}} // namespace