15
#include "convolutiondepthwise.h"
17
#include "layer_type.h"
19
#include "fused_activation.h"
23
ConvolutionDepthWise::ConvolutionDepthWise()
26
support_inplace = false;
29
int ConvolutionDepthWise::load_param(const ParamDict& pd)
31
num_output = pd.get(0, 0);
32
kernel_w = pd.get(1, 0);
33
kernel_h = pd.get(11, kernel_w);
34
dilation_w = pd.get(2, 1);
35
dilation_h = pd.get(12, dilation_w);
36
stride_w = pd.get(3, 1);
37
stride_h = pd.get(13, stride_w);
38
pad_left = pd.get(4, 0);
39
pad_right = pd.get(15, pad_left);
40
pad_top = pd.get(14, pad_left);
41
pad_bottom = pd.get(16, pad_top);
42
pad_value = pd.get(18, 0.f);
43
bias_term = pd.get(5, 0);
44
weight_data_size = pd.get(6, 0);
46
int8_scale_term = pd.get(8, 0);
47
activation_type = pd.get(9, 0);
48
activation_params = pd.get(10, Mat());
50
dynamic_weight = pd.get(19, 0);
54
one_blob_only = false;
57
if (num_output % group != 0)
66
support_int8_storage = true;
68
NCNN_LOGE("please build ncnn with NCNN_INT8 enabled for int8 inference");
76
int ConvolutionDepthWise::load_model(const ModelBin& mb)
81
weight_data = mb.load(weight_data_size, 0);
82
if (weight_data.empty())
87
bias_data = mb.load(num_output, 1);
88
if (bias_data.empty())
93
if (int8_scale_term == 1 || int8_scale_term == 101)
95
weight_data_int8_scales = mb.load(group, 1);
96
bottom_blob_int8_scales = mb.load(1, 1);
98
float bottom_blob_int8_scale = bottom_blob_int8_scales[0];
99
bottom_blob_int8_scales = Mat(group);
100
bottom_blob_int8_scales.fill(bottom_blob_int8_scale);
102
else if (int8_scale_term == 2 || int8_scale_term == 102)
104
weight_data_int8_scales = mb.load(1, 1);
105
bottom_blob_int8_scales = mb.load(1, 1);
108
float weight_data_int8_scale = weight_data_int8_scales[0];
109
weight_data_int8_scales = Mat(group);
110
weight_data_int8_scales.fill(weight_data_int8_scale);
112
float bottom_blob_int8_scale = bottom_blob_int8_scales[0];
113
bottom_blob_int8_scales = Mat(group);
114
bottom_blob_int8_scales.fill(bottom_blob_int8_scale);
117
if (int8_scale_term > 100)
119
top_blob_int8_scales = mb.load(1, 1);
121
float top_blob_int8_scale = top_blob_int8_scales[0];
122
top_blob_int8_scales = Mat(group);
123
top_blob_int8_scales.fill(top_blob_int8_scale);
129
if (weight_data.elemsize == (size_t)4u && int8_scale_term)
131
Mat int8_weight_data(weight_data_size, (size_t)1u);
132
if (int8_weight_data.empty())
135
const int weight_data_size_g = weight_data_size / group;
137
for (int g = 0; g < group; g++)
140
opt_q.num_threads = 1;
141
opt_q.blob_allocator = int8_weight_data.allocator;
142
opt_q.use_packing_layout = false;
144
const Mat weight_data_g = weight_data.range(weight_data_size_g * g, weight_data_size_g);
145
Mat int8_weight_data_g = int8_weight_data.range(weight_data_size_g * g, weight_data_size_g);
146
const Mat weight_data_int8_scales_g = weight_data_int8_scales.range(g, 1);
147
quantize_to_int8(weight_data_g, int8_weight_data_g, weight_data_int8_scales_g, opt_q);
150
weight_data = int8_weight_data;
157
static int convolutiondepthwise(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data, const Mat& bias_data, int kernel_w, int kernel_h, int stride_w, int stride_h, int dilation_w, int dilation_h, int group, int activation_type, const Mat& activation_params, const Option& opt)
159
const int w = bottom_blob.w;
160
const int inch = bottom_blob.c;
162
const int outw = top_blob.w;
163
const int outh = top_blob.h;
164
const int outch = top_blob.c;
166
const int bias_term = bias_data.empty() ? 0 : 1;
168
const int maxk = kernel_w * kernel_h;
171
std::vector<int> _space_ofs(maxk);
172
int* space_ofs = &_space_ofs[0];
176
int gap = w * dilation_h - kernel_w * dilation_w;
177
for (int i = 0; i < kernel_h; i++)
179
for (int j = 0; j < kernel_w; j++)
190
if (inch == group && group == outch)
192
#pragma omp parallel for num_threads(opt.num_threads)
193
for (int g = 0; g < group; g++)
195
float* outptr = top_blob.channel(g);
196
const float* kptr = (const float*)weight_data + maxk * g;
197
const Mat m = bottom_blob.channel(g);
199
for (int i = 0; i < outh; i++)
201
for (int j = 0; j < outw; j++)
208
const float* sptr = m.row(i * stride_h) + j * stride_w;
210
for (int k = 0; k < maxk; k++)
212
float val = sptr[space_ofs[k]];
217
outptr[j] = activation_ss(sum, activation_type, activation_params);
227
const int inch_g = inch / group;
228
const int outch_g = outch / group;
231
#pragma omp parallel for num_threads(opt.num_threads)
233
#pragma omp parallel for collapse(2) num_threads(opt.num_threads)
235
for (int g = 0; g < group; g++)
237
for (int p = 0; p < outch_g; p++)
239
float* outptr = top_blob.channel(g * outch_g + p);
240
const float* weight_data_ptr = (const float*)weight_data + maxk * inch_g * outch_g * g;
243
const int outw = top_blob.w;
244
const int outh = top_blob.h;
246
for (int i = 0; i < outh; i++)
248
for (int j = 0; j < outw; j++)
253
sum = bias_data[outch_g * g + p];
255
const float* kptr = weight_data_ptr + maxk * inch_g * p;
257
for (int q = 0; q < inch_g; q++)
259
const Mat m = bottom_blob.channel(inch_g * g + q);
260
const float* sptr = m.row(i * stride_h) + j * stride_w;
262
for (int k = 0; k < maxk; k++)
264
float val = sptr[space_ofs[k]];
272
outptr[j] = activation_ss(sum, activation_type, activation_params);
284
int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
290
if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
292
return forward_int8(bottom_blob, top_blob, opt);
298
Mat bottom_blob_bordered;
299
make_padding(bottom_blob, bottom_blob_bordered, opt);
300
if (bottom_blob_bordered.empty())
303
const int w = bottom_blob_bordered.w;
304
const int h = bottom_blob_bordered.h;
305
const size_t elemsize = bottom_blob_bordered.elemsize;
307
const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
308
const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
310
const int outw = (w - kernel_extent_w) / stride_w + 1;
311
const int outh = (h - kernel_extent_h) / stride_h + 1;
313
top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
314
if (top_blob.empty())
317
int ret = convolutiondepthwise(bottom_blob_bordered, top_blob, weight_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, dilation_w, dilation_h, group, activation_type, activation_params, opt);
324
int ConvolutionDepthWise::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
326
const Mat& bottom_blob = bottom_blobs[0];
327
const Mat& _weight_data = bottom_blobs[1];
328
Mat& top_blob = top_blobs[0];
330
const int _kernel_w = _weight_data.w;
331
const int _kernel_h = _weight_data.h;
332
const int _num_output = _weight_data.c;
334
Mat weight_data_flattened;
335
flatten(_weight_data, weight_data_flattened, opt);
336
if (weight_data_flattened.empty())
339
Mat bias_data_flattened;
342
const Mat& _bias_data = bottom_blobs[2];
343
flatten(_bias_data, bias_data_flattened, opt);
344
if (bias_data_flattened.empty())
348
Mat bottom_blob_bordered;
349
make_padding(bottom_blob, bottom_blob_bordered, _kernel_w, _kernel_h, opt);
350
if (bottom_blob_bordered.empty())
353
const int w = bottom_blob_bordered.w;
354
const int h = bottom_blob_bordered.h;
355
const size_t elemsize = bottom_blob_bordered.elemsize;
357
const int kernel_extent_w = dilation_w * (_kernel_w - 1) + 1;
358
const int kernel_extent_h = dilation_h * (_kernel_h - 1) + 1;
360
const int outw = (w - kernel_extent_w) / stride_w + 1;
361
const int outh = (h - kernel_extent_h) / stride_h + 1;
363
top_blob.create(outw, outh, _num_output, elemsize, opt.blob_allocator);
364
if (top_blob.empty())
367
int ret = convolutiondepthwise(bottom_blob_bordered, top_blob, weight_data_flattened, bias_data_flattened, _kernel_w, _kernel_h, stride_w, stride_h, dilation_w, dilation_h, group, activation_type, activation_params, opt);
374
void ConvolutionDepthWise::make_padding(const Mat& bottom_blob, Mat& bottom_blob_bordered, const Option& opt) const
376
make_padding(bottom_blob, bottom_blob_bordered, kernel_w, kernel_h, opt);
379
void ConvolutionDepthWise::make_padding(const Mat& bottom_blob, Mat& bottom_blob_bordered, int _kernel_w, int _kernel_h, const Option& opt) const
381
int w = bottom_blob.w;
382
int h = bottom_blob.h;
384
const int kernel_extent_w = dilation_w * (_kernel_w - 1) + 1;
385
const int kernel_extent_h = dilation_h * (_kernel_h - 1) + 1;
387
bottom_blob_bordered = bottom_blob;
388
if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
391
opt_b.blob_allocator = opt.workspace_allocator;
392
copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
394
else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
397
int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
398
int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
399
if (wpad > 0 || hpad > 0)
402
opt_b.blob_allocator = opt.workspace_allocator;
403
copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
406
else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
409
int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
410
int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
411
if (wpad > 0 || hpad > 0)
414
opt_b.blob_allocator = opt.workspace_allocator;
415
copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
421
static inline signed char float2int8(float v)
423
int int32 = static_cast<int>(round(v));
424
if (int32 > 127) return 127;
425
if (int32 < -127) return -127;
426
return (signed char)int32;
429
int ConvolutionDepthWise::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
434
int w = bottom_blob.w;
435
int h = bottom_blob.h;
436
int channels = bottom_blob.c;
437
size_t elemsize = bottom_blob.elemsize;
439
if (channels % group != 0 || num_output % group != 0)
447
const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
448
const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
450
Mat bottom_blob_int8 = bottom_blob;
453
const int channels_g = channels / group;
455
Mat scales(channels);
458
for (int g = 0; g < group; g++)
460
float scale = bottom_blob_int8_scales[g];
461
for (int q = 0; q < channels_g; q++)
469
opt_q.blob_allocator = opt.workspace_allocator;
470
quantize_to_int8(bottom_blob, bottom_blob_int8, scales, opt_q);
473
Mat bottom_blob_bordered;
474
make_padding(bottom_blob_int8, bottom_blob_bordered, opt);
475
if (bottom_blob_bordered.empty())
478
w = bottom_blob_bordered.w;
479
h = bottom_blob_bordered.h;
481
int outw = (w - kernel_extent_w) / stride_w + 1;
482
int outh = (h - kernel_extent_h) / stride_h + 1;
484
const int maxk = kernel_w * kernel_h;
487
std::vector<int> _space_ofs(maxk);
488
int* space_ofs = &_space_ofs[0];
492
int gap = w * dilation_h - kernel_w * dilation_w;
493
for (int i = 0; i < kernel_h; i++)
495
for (int j = 0; j < kernel_w; j++)
506
bool use_int8_requantize = int8_scale_term > 100;
507
size_t out_elemsize = use_int8_requantize ? 1u : 4u;
509
top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);
510
if (top_blob.empty())
514
if (channels == group && group == num_output)
516
#pragma omp parallel for num_threads(opt.num_threads)
517
for (int g = 0; g < group; g++)
519
signed char* outptr = top_blob.channel(g);
520
const signed char* kptr = (const signed char*)weight_data + maxk * g;
521
const Mat m = bottom_blob_bordered.channel(g);
523
for (int i = 0; i < outh; i++)
525
for (int j = 0; j < outw; j++)
529
const signed char* sptr = m.row<signed char>(i * stride_h) + j * stride_w;
531
for (int k = 0; k < maxk; k++)
533
signed char val = sptr[space_ofs[k]];
534
signed char w = kptr[k];
539
if (weight_data_int8_scales[g] == 0)
542
scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
544
float sumfp32 = sum * scale_in;
547
sumfp32 += bias_data[g];
549
sumfp32 = activation_ss(sumfp32, activation_type, activation_params);
551
if (use_int8_requantize)
554
float scale_out = top_blob_int8_scales[g];
555
signed char sums8 = float2int8(sumfp32 * scale_out);
562
((float*)outptr)[0] = sumfp32;
572
const int channels_g = channels / group;
573
const int num_output_g = num_output / group;
576
#pragma omp parallel for num_threads(opt.num_threads)
578
#pragma omp parallel for collapse(2) num_threads(opt.num_threads)
580
for (int g = 0; g < group; g++)
582
for (int p = 0; p < num_output_g; p++)
584
signed char* outptr = top_blob.channel(g * num_output_g + p);
585
const signed char* weight_data_ptr = (const signed char*)weight_data + maxk * channels_g * num_output_g * g;
587
for (int i = 0; i < outh; i++)
589
for (int j = 0; j < outw; j++)
593
const signed char* kptr = weight_data_ptr + maxk * channels_g * p;
596
for (int q = 0; q < channels_g; q++)
598
const Mat m = bottom_blob_bordered.channel(channels_g * g + q);
599
const signed char* sptr = m.row<signed char>(i * stride_h) + j * stride_w;
601
for (int k = 0; k < maxk; k++)
603
signed char val = sptr[space_ofs[k]];
604
signed char w = kptr[k];
612
if (weight_data_int8_scales[g] == 0)
615
scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
617
float sumfp32 = sum * scale_in;
620
sumfp32 += bias_data[g * num_output_g + p];
622
sumfp32 = activation_ss(sumfp32, activation_type, activation_params);
624
if (use_int8_requantize)
627
float scale_out = top_blob_int8_scales[g];
628
signed char sums8 = float2int8(sumfp32 * scale_out);
635
((float*)outptr)[0] = sumfp32;