ncnn

1
// Tencent is pleased to support the open source community by making ncnn available.
2
//
3
// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
4
//
5
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6
// in compliance with the License. You may obtain a copy of the License at
7
//
8
// https://opensource.org/licenses/BSD-3-Clause
9
//
10
// Unless required by applicable law or agreed to in writing, software distributed
11
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
13
// specific language governing permissions and limitations under the License.
14

15
#include "binaryop_vulkan.h"
16

17
#include "layer_shader_type.h"
18

19
namespace ncnn {
20

21
BinaryOp_vulkan::BinaryOp_vulkan()
22
{
23
    support_vulkan = true;
24
    support_image_storage = true;
25

26
    pipeline_binaryop = 0;
27
    pipeline_binaryop_pack4 = 0;
28
    pipeline_binaryop_pack8 = 0;
29

30
    pipeline_binaryop_broadcast[0] = 0;
31
    pipeline_binaryop_broadcast[1] = 0;
32
    pipeline_binaryop_broadcast_pack4[0] = 0;
33
    pipeline_binaryop_broadcast_pack4[1] = 0;
34
    pipeline_binaryop_broadcast_pack1to4[0] = 0;
35
    pipeline_binaryop_broadcast_pack1to4[1] = 0;
36
    pipeline_binaryop_broadcast_pack8[0] = 0;
37
    pipeline_binaryop_broadcast_pack8[1] = 0;
38
    pipeline_binaryop_broadcast_pack1to8[0] = 0;
39
    pipeline_binaryop_broadcast_pack1to8[1] = 0;
40
}
41

42
static int get_reverse_op_type(int op_type)
43
{
44
    if (op_type == BinaryOp::Operation_SUB) return BinaryOp::Operation_RSUB;
45
    if (op_type == BinaryOp::Operation_DIV) return BinaryOp::Operation_RDIV;
46
    if (op_type == BinaryOp::Operation_POW) return BinaryOp::Operation_RPOW;
47
    if (op_type == BinaryOp::Operation_ATAN2) return BinaryOp::Operation_RATAN2;
48
    if (op_type == BinaryOp::Operation_RSUB) return BinaryOp::Operation_SUB;
49
    if (op_type == BinaryOp::Operation_RDIV) return BinaryOp::Operation_DIV;
50
    if (op_type == BinaryOp::Operation_RPOW) return BinaryOp::Operation_POW;
51
    if (op_type == BinaryOp::Operation_RATAN2) return BinaryOp::Operation_ATAN2;
52
    return op_type;
53
}
54

55
int BinaryOp_vulkan::create_pipeline(const Option& opt)
56
{
57
    const Mat& A_shape = bottom_shapes.empty() ? Mat() : bottom_shapes[0];
58
    const Mat& B_shape = with_scalar ? A_shape : bottom_shapes.empty() ? Mat() : bottom_shapes[1];
59
    const Mat& out_shape = top_shapes.empty() ? Mat() : top_shapes[0];
60

61
    int A_elempack = 1;
62
    if (A_shape.dims == 1) A_elempack = opt.use_shader_pack8 && A_shape.w % 8 == 0 ? 8 : A_shape.w % 4 == 0 ? 4 : 1;
63
    if (A_shape.dims == 2) A_elempack = opt.use_shader_pack8 && A_shape.h % 8 == 0 ? 8 : A_shape.h % 4 == 0 ? 4 : 1;
64
    if (A_shape.dims == 3 || A_shape.dims == 4) A_elempack = opt.use_shader_pack8 && A_shape.c % 8 == 0 ? 8 : A_shape.c % 4 == 0 ? 4 : 1;
65

66
    int B_elempack = 1;
67
    if (B_shape.dims == 1) B_elempack = opt.use_shader_pack8 && B_shape.w % 8 == 0 ? 8 : B_shape.w % 4 == 0 ? 4 : 1;
68
    if (B_shape.dims == 2) B_elempack = opt.use_shader_pack8 && B_shape.h % 8 == 0 ? 8 : B_shape.h % 4 == 0 ? 4 : 1;
69
    if (B_shape.dims == 3 || B_shape.dims == 4) B_elempack = opt.use_shader_pack8 && B_shape.c % 8 == 0 ? 8 : B_shape.c % 4 == 0 ? 4 : 1;
70

71
    int out_elempack = 1;
72
    if (out_shape.dims == 1) out_elempack = opt.use_shader_pack8 && out_shape.w % 8 == 0 ? 8 : out_shape.w % 4 == 0 ? 4 : 1;
73
    if (out_shape.dims == 2) out_elempack = opt.use_shader_pack8 && out_shape.h % 8 == 0 ? 8 : out_shape.h % 4 == 0 ? 4 : 1;
74
    if (out_shape.dims == 3 || out_shape.dims == 4) out_elempack = opt.use_shader_pack8 && out_shape.c % 8 == 0 ? 8 : out_shape.c % 4 == 0 ? 4 : 1;
75

76
    size_t A_elemsize;
77
    size_t B_elemsize;
78
    size_t out_elemsize;
79
    if (opt.use_fp16_storage)
80
    {
81
        A_elemsize = A_elempack * 2u;
82
        B_elemsize = B_elempack * 2u;
83
        out_elemsize = out_elempack * 2u;
84
    }
85
    else if (opt.use_fp16_packed)
86
    {
87
        A_elemsize = A_elempack == 1 ? 4u : A_elempack * 2u;
88
        B_elemsize = B_elempack == 1 ? 4u : B_elempack * 2u;
89
        out_elemsize = out_elempack == 1 ? 4u : out_elempack * 2u;
90
    }
91
    else
92
    {
93
        A_elemsize = A_elempack * 4u;
94
        B_elemsize = B_elempack * 4u;
95
        out_elemsize = out_elempack * 4u;
96
    }
97

98
    Mat A_shape_packed;
99
    if (A_shape.dims == 1) A_shape_packed = Mat(A_shape.w / A_elempack, (void*)0, A_elemsize, A_elempack);
100
    if (A_shape.dims == 2) A_shape_packed = Mat(A_shape.w, A_shape.h / A_elempack, (void*)0, A_elemsize, A_elempack);
101
    if (A_shape.dims == 3) A_shape_packed = Mat(A_shape.w, A_shape.h, A_shape.c / A_elempack, (void*)0, A_elemsize, A_elempack);
102
    if (A_shape.dims == 4) A_shape_packed = Mat(A_shape.w, A_shape.h, A_shape.d, A_shape.c / A_elempack, (void*)0, A_elemsize, A_elempack);
103

104
    Mat B_shape_packed;
105
    if (B_shape.dims == 1) B_shape_packed = Mat(B_shape.w / B_elempack, (void*)0, B_elemsize, B_elempack);
106
    if (B_shape.dims == 2) B_shape_packed = Mat(B_shape.w, B_shape.h / B_elempack, (void*)0, B_elemsize, B_elempack);
107
    if (B_shape.dims == 3) B_shape_packed = Mat(B_shape.w, B_shape.h, B_shape.c / B_elempack, (void*)0, B_elemsize, B_elempack);
108
    if (B_shape.dims == 4) B_shape_packed = Mat(B_shape.w, B_shape.h, B_shape.d, B_shape.c / B_elempack, (void*)0, B_elemsize, B_elempack);
109

110
    Mat out_shape_packed;
111
    if (out_shape.dims == 1) out_shape_packed = Mat(out_shape.w / out_elempack, (void*)0, out_elemsize, out_elempack);
112
    if (out_shape.dims == 2) out_shape_packed = Mat(out_shape.w, out_shape.h / out_elempack, (void*)0, out_elemsize, out_elempack);
113
    if (out_shape.dims == 3) out_shape_packed = Mat(out_shape.w, out_shape.h, out_shape.c / out_elempack, (void*)0, out_elemsize, out_elempack);
114
    if (out_shape.dims == 4) out_shape_packed = Mat(out_shape.w, out_shape.h, out_shape.d, out_shape.c / out_elempack, (void*)0, out_elemsize, out_elempack);
115

116
    bool broadcast = true;
117
    if (A_shape.dims == B_shape.dims && A_shape.w == B_shape.w && A_shape.h == B_shape.h && A_shape.d == B_shape.d && A_shape.c == B_shape.c)
118
    {
119
        broadcast = false;
120
    }
121

122
    // no broadcast
123
    if (out_shape.dims == 0 || !broadcast)
124
    {
125
        std::vector<vk_specialization_type> specializations(3 + 15);
126
        specializations[0].i = op_type;
127
        specializations[1].i = with_scalar;
128
        specializations[2].f = b;
129
        specializations[3 + 0].i = A_shape_packed.dims;
130
        specializations[3 + 1].i = A_shape_packed.w;
131
        specializations[3 + 2].i = A_shape_packed.h * A_shape_packed.d;
132
        specializations[3 + 3].i = A_shape_packed.c;
133
        specializations[3 + 4].i = A_shape_packed.cstep;
134
        specializations[3 + 5].i = B_shape_packed.dims;
135
        specializations[3 + 6].i = B_shape_packed.w;
136
        specializations[3 + 7].i = B_shape_packed.h * B_shape_packed.d;
137
        specializations[3 + 8].i = B_shape_packed.c;
138
        specializations[3 + 9].i = B_shape_packed.cstep;
139
        specializations[3 + 10].i = out_shape_packed.dims;
140
        specializations[3 + 11].i = out_shape_packed.w;
141
        specializations[3 + 12].i = out_shape_packed.h * out_shape_packed.d;
142
        specializations[3 + 13].i = out_shape_packed.c;
143
        specializations[3 + 14].i = out_shape_packed.cstep;
144

145
        Mat local_size_xyz;
146
        if (out_shape_packed.dims == 1)
147
        {
148
            local_size_xyz.w = std::min(64, out_shape_packed.w);
149
            local_size_xyz.h = 1;
150
            local_size_xyz.c = 1;
151
        }
152
        if (out_shape_packed.dims == 2)
153
        {
154
            local_size_xyz.w = std::min(8, out_shape_packed.w);
155
            local_size_xyz.h = std::min(8, out_shape_packed.h);
156
            local_size_xyz.c = 1;
157
        }
158
        if (out_shape_packed.dims == 3)
159
        {
160
            local_size_xyz.w = std::min(4, out_shape_packed.w);
161
            local_size_xyz.h = std::min(4, out_shape_packed.h);
162
            local_size_xyz.c = std::min(4, out_shape_packed.c);
163
        }
164
        if (out_shape_packed.dims == 4)
165
        {
166
            local_size_xyz.w = std::min(4, out_shape_packed.w);
167
            local_size_xyz.h = std::min(4, out_shape_packed.h * out_shape_packed.d);
168
            local_size_xyz.c = std::min(4, out_shape_packed.c);
169
        }
170

171
        // pack1
172
        if (out_shape.dims == 0 || out_elempack == 1)
173
        {
174
            pipeline_binaryop = new Pipeline(vkdev);
175
            pipeline_binaryop->set_optimal_local_size_xyz(local_size_xyz);
176
            pipeline_binaryop->create(LayerShaderType::binaryop, opt, specializations);
177
        }
178

179
        // pack4
180
        if (out_shape.dims == 0 || out_elempack == 4)
181
        {
182
            pipeline_binaryop_pack4 = new Pipeline(vkdev);
183
            pipeline_binaryop_pack4->set_optimal_local_size_xyz(local_size_xyz);
184
            pipeline_binaryop_pack4->create(LayerShaderType::binaryop_pack4, opt, specializations);
185
        }
186

187
        // pack8
188
        if ((opt.use_shader_pack8 && out_shape.dims == 0) || out_elempack == 8)
189
        {
190
            pipeline_binaryop_pack8 = new Pipeline(vkdev);
191
            pipeline_binaryop_pack8->set_optimal_local_size_xyz(local_size_xyz);
192
            pipeline_binaryop_pack8->create(LayerShaderType::binaryop_pack8, opt, specializations);
193
        }
194
    }
195

196
    // broadcast
197
    if (out_shape.dims == 0 || broadcast)
198
    {
199
        if (A_shape.dims > 0 && B_shape.dims > 0)
200
        {
201
            const bool a_rank_is_lower = A_shape.dims < B_shape.dims;
202
            const bool a_rank_is_equal = A_shape.dims == B_shape.dims;
203
            const bool a_pack_is_lower = A_elempack < B_elempack;
204
            const bool a_pack_is_equal = A_elempack == B_elempack;
205
            const bool a_size_is_lower = A_shape.w * A_shape.h * A_shape.d * A_shape.c < B_shape.w * B_shape.h * B_shape.d * B_shape.c;
206
            if (a_rank_is_lower || (a_rank_is_equal && a_pack_is_lower) || (a_pack_is_equal && a_size_is_lower))
207
            {
208
                // swap AB
209
                std::swap(A_shape_packed, B_shape_packed);
210
            }
211

212
            if (B_shape_packed.dims == 1 && ((A_shape_packed.dims == 2 && B_shape_packed.w * B_shape_packed.elempack != A_shape_packed.h * A_shape_packed.elempack) || ((A_shape_packed.dims == 3 || A_shape_packed.dims == 4) && B_shape_packed.w * B_shape_packed.elempack != A_shape_packed.c * A_shape_packed.elempack)))
213
            {
214
                B_shape_packed.dims = out_shape.dims;
215
                B_shape_packed.w = B_shape_packed.w * B_shape_packed.elempack;
216
                B_shape_packed.elempack = 1;
217
            }
218
        }
219

220
        const int op_type_r = get_reverse_op_type(op_type);
221

222
        std::vector<vk_specialization_type> specializations(1 + 18);
223
        specializations[0].i = op_type;
224
        specializations[1 + 0].i = A_shape_packed.dims;
225
        specializations[1 + 1].i = A_shape_packed.w;
226
        specializations[1 + 2].i = A_shape_packed.h;
227
        specializations[1 + 3].i = A_shape_packed.d;
228
        specializations[1 + 4].i = A_shape_packed.c;
229
        specializations[1 + 5].i = A_shape_packed.cstep;
230
        specializations[1 + 6].i = B_shape_packed.dims;
231
        specializations[1 + 7].i = B_shape_packed.w;
232
        specializations[1 + 8].i = B_shape_packed.h;
233
        specializations[1 + 9].i = B_shape_packed.d;
234
        specializations[1 + 10].i = B_shape_packed.c;
235
        specializations[1 + 11].i = B_shape_packed.cstep;
236
        specializations[1 + 12].i = out_shape_packed.dims;
237
        specializations[1 + 13].i = out_shape_packed.w;
238
        specializations[1 + 14].i = out_shape_packed.h;
239
        specializations[1 + 15].i = out_shape_packed.d;
240
        specializations[1 + 16].i = out_shape_packed.c;
241
        specializations[1 + 17].i = out_shape_packed.cstep;
242

243
        std::vector<vk_specialization_type> specializations_r(1 + 18);
244
        specializations_r[0].i = op_type_r;
245
        specializations_r[1 + 0].i = A_shape_packed.dims;
246
        specializations_r[1 + 1].i = A_shape_packed.w;
247
        specializations_r[1 + 2].i = A_shape_packed.h;
248
        specializations_r[1 + 3].i = A_shape_packed.d;
249
        specializations_r[1 + 4].i = A_shape_packed.c;
250
        specializations_r[1 + 5].i = A_shape_packed.cstep;
251
        specializations_r[1 + 6].i = B_shape_packed.dims;
252
        specializations_r[1 + 7].i = B_shape_packed.w;
253
        specializations_r[1 + 8].i = B_shape_packed.h;
254
        specializations_r[1 + 9].i = B_shape_packed.d;
255
        specializations_r[1 + 10].i = B_shape_packed.c;
256
        specializations_r[1 + 11].i = B_shape_packed.cstep;
257
        specializations_r[1 + 12].i = out_shape_packed.dims;
258
        specializations_r[1 + 13].i = out_shape_packed.w;
259
        specializations_r[1 + 14].i = out_shape_packed.h;
260
        specializations_r[1 + 15].i = out_shape_packed.d;
261
        specializations_r[1 + 16].i = out_shape_packed.c;
262
        specializations_r[1 + 17].i = out_shape_packed.cstep;
263

264
        Mat local_size_xyz;
265
        if (out_shape_packed.dims == 1)
266
        {
267
            local_size_xyz.w = std::min(64, out_shape_packed.w);
268
            local_size_xyz.h = 1;
269
            local_size_xyz.c = 1;
270
        }
271
        if (out_shape_packed.dims == 2)
272
        {
273
            local_size_xyz.w = std::min(8, out_shape_packed.w);
274
            local_size_xyz.h = std::min(8, out_shape_packed.h);
275
            local_size_xyz.c = 1;
276
        }
277
        if (out_shape_packed.dims == 3)
278
        {
279
            local_size_xyz.w = std::min(4, out_shape_packed.w);
280
            local_size_xyz.h = std::min(4, out_shape_packed.h);
281
            local_size_xyz.c = std::min(4, out_shape_packed.c);
282
        }
283
        if (out_shape_packed.dims == 4)
284
        {
285
            local_size_xyz.w = std::min(4, out_shape_packed.w);
286
            local_size_xyz.h = std::min(4, out_shape_packed.h * out_shape_packed.d);
287
            local_size_xyz.c = std::min(4, out_shape_packed.c);
288
        }
289

290
        // pack1
291
        if (out_shape.dims == 0 || (out_elempack == 1))
292
        {
293
            pipeline_binaryop_broadcast[0] = new Pipeline(vkdev);
294
            pipeline_binaryop_broadcast[0]->set_optimal_local_size_xyz(local_size_xyz);
295
            pipeline_binaryop_broadcast[0]->create(LayerShaderType::binaryop_broadcast, opt, specializations);
296

297
            if (op_type_r != op_type)
298
            {
299
                // sub div pow ...
300
                pipeline_binaryop_broadcast[1] = new Pipeline(vkdev);
301
                pipeline_binaryop_broadcast[1]->set_optimal_local_size_xyz(local_size_xyz);
302
                pipeline_binaryop_broadcast[1]->create(LayerShaderType::binaryop_broadcast, opt, specializations_r);
303
            }
304
        }
305

306
        // pack4
307
        if (out_shape.dims == 0 || (A_shape_packed.elempack == 4 && B_shape_packed.elempack == 4 && out_elempack == 4))
308
        {
309
            pipeline_binaryop_broadcast_pack4[0] = new Pipeline(vkdev);
310
            pipeline_binaryop_broadcast_pack4[0]->set_optimal_local_size_xyz(local_size_xyz);
311
            pipeline_binaryop_broadcast_pack4[0]->create(LayerShaderType::binaryop_broadcast_pack4, opt, specializations);
312

313
            if (op_type_r != op_type)
314
            {
315
                // sub div pow ...
316
                pipeline_binaryop_broadcast_pack4[1] = new Pipeline(vkdev);
317
                pipeline_binaryop_broadcast_pack4[1]->set_optimal_local_size_xyz(local_size_xyz);
318
                pipeline_binaryop_broadcast_pack4[1]->create(LayerShaderType::binaryop_broadcast_pack4, opt, specializations_r);
319
            }
320
        }
321

322
        // pack1to4
323
        if (out_shape.dims == 0 || ((A_shape_packed.elempack == 1 || B_shape_packed.elempack == 1) && out_elempack == 4))
324
        {
325
            pipeline_binaryop_broadcast_pack1to4[0] = new Pipeline(vkdev);
326
            pipeline_binaryop_broadcast_pack1to4[0]->set_optimal_local_size_xyz(local_size_xyz);
327
            pipeline_binaryop_broadcast_pack1to4[0]->create(LayerShaderType::binaryop_broadcast_pack1to4, opt, specializations);
328

329
            if (op_type_r != op_type)
330
            {
331
                // sub div pow ...
332
                pipeline_binaryop_broadcast_pack1to4[1] = new Pipeline(vkdev);
333
                pipeline_binaryop_broadcast_pack1to4[1]->set_optimal_local_size_xyz(local_size_xyz);
334
                pipeline_binaryop_broadcast_pack1to4[1]->create(LayerShaderType::binaryop_broadcast_pack1to4, opt, specializations_r);
335
            }
336
        }
337

338
        // pack8
339
        if ((opt.use_shader_pack8 && out_shape.dims == 0) || (A_shape_packed.elempack == 8 && B_shape_packed.elempack == 8 && out_elempack == 8))
340
        {
341
            pipeline_binaryop_broadcast_pack8[0] = new Pipeline(vkdev);
342
            pipeline_binaryop_broadcast_pack8[0]->set_optimal_local_size_xyz(local_size_xyz);
343
            pipeline_binaryop_broadcast_pack8[0]->create(LayerShaderType::binaryop_broadcast_pack8, opt, specializations);
344

345
            if (op_type_r != op_type)
346
            {
347
                // sub div pow ...
348
                pipeline_binaryop_broadcast_pack8[1] = new Pipeline(vkdev);
349
                pipeline_binaryop_broadcast_pack8[1]->set_optimal_local_size_xyz(local_size_xyz);
350
                pipeline_binaryop_broadcast_pack8[1]->create(LayerShaderType::binaryop_broadcast_pack8, opt, specializations_r);
351
            }
352
        }
353

354
        // pack1to8
355
        if ((opt.use_shader_pack8 && out_shape.dims == 0) || ((A_shape_packed.elempack == 1 || B_shape_packed.elempack == 1) && out_elempack == 8))
356
        {
357
            pipeline_binaryop_broadcast_pack1to8[0] = new Pipeline(vkdev);
358
            pipeline_binaryop_broadcast_pack1to8[0]->set_optimal_local_size_xyz(local_size_xyz);
359
            pipeline_binaryop_broadcast_pack1to8[0]->create(LayerShaderType::binaryop_broadcast_pack1to8, opt, specializations);
360

361
            if (op_type_r != op_type)
362
            {
363
                // sub div pow ...
364
                pipeline_binaryop_broadcast_pack1to8[1] = new Pipeline(vkdev);
365
                pipeline_binaryop_broadcast_pack1to8[1]->set_optimal_local_size_xyz(local_size_xyz);
366
                pipeline_binaryop_broadcast_pack1to8[1]->create(LayerShaderType::binaryop_broadcast_pack1to8, opt, specializations_r);
367
            }
368
        }
369
    }
370

371
    return 0;
372
}
373

374
int BinaryOp_vulkan::destroy_pipeline(const Option& /*opt*/)
375
{
376
    delete pipeline_binaryop;
377
    pipeline_binaryop = 0;
378

379
    delete pipeline_binaryop_pack4;
380
    pipeline_binaryop_pack4 = 0;
381

382
    delete pipeline_binaryop_pack8;
383
    pipeline_binaryop_pack8 = 0;
384

385
    delete pipeline_binaryop_broadcast[0];
386
    delete pipeline_binaryop_broadcast[1];
387
    pipeline_binaryop_broadcast[0] = 0;
388
    pipeline_binaryop_broadcast[1] = 0;
389

390
    delete pipeline_binaryop_broadcast_pack4[0];
391
    delete pipeline_binaryop_broadcast_pack4[1];
392
    pipeline_binaryop_broadcast_pack4[0] = 0;
393
    pipeline_binaryop_broadcast_pack4[1] = 0;
394

395
    delete pipeline_binaryop_broadcast_pack1to4[0];
396
    delete pipeline_binaryop_broadcast_pack1to4[1];
397
    pipeline_binaryop_broadcast_pack1to4[0] = 0;
398
    pipeline_binaryop_broadcast_pack1to4[1] = 0;
399

400
    delete pipeline_binaryop_broadcast_pack1to8[0];
401
    delete pipeline_binaryop_broadcast_pack1to8[1];
402
    pipeline_binaryop_broadcast_pack1to8[0] = 0;
403
    pipeline_binaryop_broadcast_pack1to8[1] = 0;
404

405
    delete pipeline_binaryop_broadcast_pack1to8[0];
406
    delete pipeline_binaryop_broadcast_pack1to8[1];
407
    pipeline_binaryop_broadcast_pack1to8[0] = 0;
408
    pipeline_binaryop_broadcast_pack1to8[1] = 0;
409

410
    return 0;
411
}
412

413
int BinaryOp_vulkan::forward(const std::vector<VkMat>& bottom_blobs, std::vector<VkMat>& top_blobs, VkCompute& cmd, const Option& opt) const
414
{
415
    const VkMat& A = bottom_blobs[0];
416
    const VkMat& B = bottom_blobs[1];
417
    const int outdims = std::max(A.dims, B.dims);
418

419
    const bool a_rank_is_lower = A.dims < B.dims;
420
    const bool b_rank_is_lower = B.dims < A.dims;
421
    const bool a_rank_is_equal = A.dims == B.dims;
422

423
    VkMat& top_blob = top_blobs[0];
424
    if (a_rank_is_lower)
425
    {
426
        top_blob.create_like(B, opt.blob_vkallocator);
427
    }
428
    else if (b_rank_is_lower)
429
    {
430
        top_blob.create_like(A, opt.blob_vkallocator);
431
    }
432
    else
433
    {
434
        const int outw = std::max(A.w, B.w);
435
        const int outh = std::max(A.h, B.h);
436
        const int outd = std::max(A.d, B.d);
437
        const int outc = std::max(A.c, B.c);
438
        const int out_elempack = std::max(A.elempack, B.elempack);
439
        const size_t out_elemsize = std::max(A.elemsize, B.elemsize);
440

441
        if (outdims == 1)
442
        {
443
            top_blob.create(outw, out_elemsize, out_elempack, opt.blob_vkallocator);
444
        }
445
        if (outdims == 2)
446
        {
447
            top_blob.create(outw, outh, out_elemsize, out_elempack, opt.blob_vkallocator);
448
        }
449
        if (outdims == 3)
450
        {
451
            top_blob.create(outw, outh, outc, out_elemsize, out_elempack, opt.blob_vkallocator);
452
        }
453
        if (outdims == 4)
454
        {
455
            top_blob.create(outw, outh, outd, outc, out_elemsize, out_elempack, opt.blob_vkallocator);
456
        }
457
    }
458
    if (top_blob.empty())
459
        return -100;
460

461
    // no broadcast
462
    if (A.dims == B.dims && A.w == B.w && A.h == B.h && A.d == B.d && A.c == B.c && A.elempack == B.elempack)
463
    {
464
        std::vector<VkMat> bindings(3);
465
        bindings[0] = A;
466
        bindings[1] = B;
467
        bindings[2] = top_blob;
468

469
        std::vector<vk_constant_type> constants(15);
470
        constants[0].i = A.dims;
471
        constants[1].i = A.w;
472
        constants[2].i = A.h * A.d;
473
        constants[3].i = A.c;
474
        constants[4].i = A.cstep;
475
        constants[5].i = B.dims;
476
        constants[6].i = B.w;
477
        constants[7].i = B.h * B.d;
478
        constants[8].i = B.c;
479
        constants[9].i = B.cstep;
480
        constants[10].i = top_blob.dims;
481
        constants[11].i = top_blob.w;
482
        constants[12].i = top_blob.h * top_blob.d;
483
        constants[13].i = top_blob.c;
484
        constants[14].i = top_blob.cstep;
485

486
        const Pipeline* pipeline = top_blob.elempack == 8 ? pipeline_binaryop_pack8
487
                                   : top_blob.elempack == 4 ? pipeline_binaryop_pack4
488
                                   : pipeline_binaryop;
489

490
        cmd.record_pipeline(pipeline, bindings, constants, top_blob);
491

492
        return 0;
493
    }
494

495
    const bool a_pack_is_lower = A.elempack < B.elempack;
496
    const bool a_pack_is_equal = A.elempack == B.elempack;
497
    const bool a_size_is_lower = A.w * A.h * A.d * A.c * A.elempack < B.w * B.h * B.d * B.c * B.elempack;
498
    if (a_rank_is_lower || (a_rank_is_equal && a_pack_is_lower) || (a_pack_is_equal && a_size_is_lower))
499
    {
500
        VkMat A2;
501
        if (A.dims == 1 && ((B.dims == 2 && A.w * A.elempack != B.h * B.elempack) || ((B.dims == 3 || B.dims == 4) && A.w * A.elempack != B.c * B.elempack)))
502
        {
503
            vkdev->convert_packing(A, A2, 1, cmd, opt);
504
            A2.dims = top_blob.dims;
505
        }
506
        else
507
        {
508
            A2 = A;
509
        }
510

511
        std::vector<VkMat> bindings(3);
512
        bindings[0] = B;
513
        bindings[1] = A2;
514
        bindings[2] = top_blob;
515

516
        std::vector<vk_constant_type> constants(18);
517
        constants[0].i = B.dims;
518
        constants[1].i = B.w;
519
        constants[2].i = B.h;
520
        constants[3].i = B.d;
521
        constants[4].i = B.c;
522
        constants[5].i = B.cstep;
523
        constants[6].i = A2.dims;
524
        constants[7].i = A2.w;
525
        constants[8].i = A2.h;
526
        constants[9].i = A2.d;
527
        constants[10].i = A2.c;
528
        constants[11].i = A2.cstep;
529
        constants[12].i = top_blob.dims;
530
        constants[13].i = top_blob.w;
531
        constants[14].i = top_blob.h;
532
        constants[15].i = top_blob.d;
533
        constants[16].i = top_blob.c;
534
        constants[17].i = top_blob.cstep;
535

536
        const int ri = get_reverse_op_type(op_type) == op_type ? 0 : 1;
537

538
        const Pipeline* pipeline = 0;
539
        if (A2.elempack == 1 && top_blob.elempack == 1)
540
        {
541
            pipeline = pipeline_binaryop_broadcast[ri];
542
        }
543
        if (A2.elempack == 4 && top_blob.elempack == 4)
544
        {
545
            pipeline = pipeline_binaryop_broadcast_pack4[ri];
546
        }
547
        if (A2.elempack == 1 && top_blob.elempack == 4)
548
        {
549
            pipeline = pipeline_binaryop_broadcast_pack1to4[ri];
550
        }
551
        if (A2.elempack == 8 && top_blob.elempack == 8)
552
        {
553
            pipeline = pipeline_binaryop_broadcast_pack8[ri];
554
        }
555
        if (A2.elempack == 1 && top_blob.elempack == 8)
556
        {
557
            pipeline = pipeline_binaryop_broadcast_pack1to8[ri];
558
        }
559

560
        cmd.record_pipeline(pipeline, bindings, constants, top_blob);
561
    }
562
    else
563
    {
564
        VkMat B2;
565
        if (B.dims == 1 && ((A.dims == 2 && B.w * B.elempack != A.h * A.elempack) || ((A.dims == 3 || A.dims == 4) && B.w * B.elempack != A.c * A.elempack)))
566
        {
567
            vkdev->convert_packing(B, B2, 1, cmd, opt);
568
            B2.dims = top_blob.dims;
569
        }
570
        else
571
        {
572
            B2 = B;
573
        }
574

575
        std::vector<VkMat> bindings(3);
576
        bindings[0] = A;
577
        bindings[1] = B2;
578
        bindings[2] = top_blob;
579

580
        std::vector<vk_constant_type> constants(18);
581
        constants[0].i = A.dims;
582
        constants[1].i = A.w;
583
        constants[2].i = A.h;
584
        constants[3].i = A.d;
585
        constants[4].i = A.c;
586
        constants[5].i = A.cstep;
587
        constants[6].i = B2.dims;
588
        constants[7].i = B2.w;
589
        constants[8].i = B2.h;
590
        constants[9].i = B2.d;
591
        constants[10].i = B2.c;
592
        constants[11].i = B2.cstep;
593
        constants[12].i = top_blob.dims;
594
        constants[13].i = top_blob.w;
595
        constants[14].i = top_blob.h;
596
        constants[15].i = top_blob.d;
597
        constants[16].i = top_blob.c;
598
        constants[17].i = top_blob.cstep;
599

600
        const Pipeline* pipeline = 0;
601
        if (B2.elempack == 1 && top_blob.elempack == 1)
602
        {
603
            pipeline = pipeline_binaryop_broadcast[0];
604
        }
605
        if (B2.elempack == 4 && top_blob.elempack == 4)
606
        {
607
            pipeline = pipeline_binaryop_broadcast_pack4[0];
608
        }
609
        if (B2.elempack == 1 && top_blob.elempack == 4)
610
        {
611
            pipeline = pipeline_binaryop_broadcast_pack1to4[0];
612
        }
613
        if (B2.elempack == 8 && top_blob.elempack == 8)
614
        {
615
            pipeline = pipeline_binaryop_broadcast_pack8[0];
616
        }
617
        if (B2.elempack == 1 && top_blob.elempack == 8)
618
        {
619
            pipeline = pipeline_binaryop_broadcast_pack1to8[0];
620
        }
621

622
        cmd.record_pipeline(pipeline, bindings, constants, top_blob);
623
    }
624

625
    return 0;
626
}
627

628
int BinaryOp_vulkan::forward_inplace(VkMat& bottom_top_blob, VkCompute& cmd, const Option& /*opt*/) const
629
{
630
    int elempack = bottom_top_blob.elempack;
631

632
    std::vector<VkMat> bindings(3);
633
    bindings[0] = bottom_top_blob;
634
    bindings[1] = bottom_top_blob; // TODO use dummy buffer
635
    bindings[2] = bottom_top_blob; // TODO use dummy buffer
636

637
    std::vector<vk_constant_type> constants(15);
638
    constants[10].i = bottom_top_blob.dims;
639
    constants[11].i = bottom_top_blob.w;
640
    constants[12].i = bottom_top_blob.h * bottom_top_blob.d;
641
    constants[13].i = bottom_top_blob.c;
642
    constants[14].i = bottom_top_blob.cstep;
643

644
    const Pipeline* pipeline = elempack == 8 ? pipeline_binaryop_pack8
645
                               : elempack == 4 ? pipeline_binaryop_pack4
646
                               : pipeline_binaryop;
647

648
    cmd.record_pipeline(pipeline, bindings, constants, bottom_top_blob);
649

650
    return 0;
651
}
652

653
int BinaryOp_vulkan::forward(const std::vector<VkImageMat>& bottom_blobs, std::vector<VkImageMat>& top_blobs, VkCompute& cmd, const Option& opt) const
654
{
655
    const VkImageMat& A = bottom_blobs[0];
656
    const VkImageMat& B = bottom_blobs[1];
657
    const int outdims = std::max(A.dims, B.dims);
658

659
    const bool a_rank_is_lower = A.dims < B.dims;
660
    const bool b_rank_is_lower = B.dims < A.dims;
661
    const bool a_rank_is_equal = A.dims == B.dims;
662

663
    VkImageMat& top_blob = top_blobs[0];
664
    if (a_rank_is_lower)
665
    {
666
        top_blob.create_like(B, opt.blob_vkallocator);
667
    }
668
    else if (b_rank_is_lower)
669
    {
670
        top_blob.create_like(A, opt.blob_vkallocator);
671
    }
672
    else
673
    {
674
        const int outw = std::max(A.w, B.w);
675
        const int outh = std::max(A.h, B.h);
676
        const int outd = std::max(A.d, B.d);
677
        const int outc = std::max(A.c, B.c);
678
        const int out_elempack = std::max(A.elempack, B.elempack);
679
        const size_t out_elemsize = std::max(A.elemsize, B.elemsize);
680

681
        if (outdims == 1)
682
        {
683
            top_blob.create(outw, out_elemsize, out_elempack, opt.blob_vkallocator);
684
        }
685
        if (outdims == 2)
686
        {
687
            top_blob.create(outw, outh, out_elemsize, out_elempack, opt.blob_vkallocator);
688
        }
689
        if (outdims == 3)
690
        {
691
            top_blob.create(outw, outh, outc, out_elemsize, out_elempack, opt.blob_vkallocator);
692
        }
693
        if (outdims == 4)
694
        {
695
            top_blob.create(outw, outh, outd, outc, out_elemsize, out_elempack, opt.blob_vkallocator);
696
        }
697
    }
698
    if (top_blob.empty())
699
        return -100;
700

701
    // no broadcast
702
    if (A.dims == B.dims && A.w == B.w && A.h == B.h && A.d == B.d && A.c == B.c && A.elempack == B.elempack)
703
    {
704
        std::vector<VkImageMat> bindings(3);
705
        bindings[0] = A;
706
        bindings[1] = B;
707
        bindings[2] = top_blob;
708

709
        std::vector<vk_constant_type> constants(15);
710
        constants[0].i = A.dims;
711
        constants[1].i = A.w;
712
        constants[2].i = A.h * A.d;
713
        constants[3].i = A.c;
714
        constants[4].i = 0; //A.cstep;
715
        constants[5].i = B.dims;
716
        constants[6].i = B.w;
717
        constants[7].i = B.h * B.d;
718
        constants[8].i = B.c;
719
        constants[9].i = 0; //B.cstep;
720
        constants[10].i = top_blob.dims;
721
        constants[11].i = top_blob.w;
722
        constants[12].i = top_blob.h * top_blob.d;
723
        constants[13].i = top_blob.c;
724
        constants[14].i = 0; //top_blob.cstep;
725

726
        const Pipeline* pipeline = top_blob.elempack == 8 ? pipeline_binaryop_pack8
727
                                   : top_blob.elempack == 4 ? pipeline_binaryop_pack4
728
                                   : pipeline_binaryop;
729

730
        cmd.record_pipeline(pipeline, bindings, constants, top_blob);
731

732
        return 0;
733
    }
734

735
    const bool a_pack_is_lower = A.elempack < B.elempack;
736
    const bool a_pack_is_equal = A.elempack == B.elempack;
737
    const bool a_size_is_lower = A.w * A.h * A.d * A.c * A.elempack < B.w * B.h * B.d * B.c * B.elempack;
738
    if (a_rank_is_lower || (a_rank_is_equal && a_pack_is_lower) || (a_pack_is_equal && a_size_is_lower))
739
    {
740
        VkImageMat A2;
741
        if (A.dims == 1 && ((B.dims == 2 && A.w * A.elempack != B.h * B.elempack) || ((B.dims == 3 || B.dims == 4) && A.w * A.elempack != B.c * B.elempack)))
742
        {
743
            vkdev->convert_packing(A, A2, 1, cmd, opt);
744
            A2.dims = top_blob.dims;
745
        }
746
        else
747
        {
748
            A2 = A;
749
        }
750

751
        std::vector<VkImageMat> bindings(3);
752
        bindings[0] = B;
753
        bindings[1] = A2;
754
        bindings[2] = top_blob;
755

756
        std::vector<vk_constant_type> constants(18);
757
        constants[0].i = B.dims;
758
        constants[1].i = B.w;
759
        constants[2].i = B.h;
760
        constants[3].i = B.d;
761
        constants[4].i = B.c;
762
        constants[5].i = 0; //B.cstep;
763
        constants[6].i = A2.dims;
764
        constants[7].i = A2.w;
765
        constants[8].i = A2.h;
766
        constants[9].i = A2.d;
767
        constants[10].i = A2.c;
768
        constants[11].i = 0; //A2.cstep;
769
        constants[12].i = top_blob.dims;
770
        constants[13].i = top_blob.w;
771
        constants[14].i = top_blob.h;
772
        constants[15].i = top_blob.d;
773
        constants[16].i = top_blob.c;
774
        constants[17].i = 0; //top_blob.cstep;
775

776
        const int ri = get_reverse_op_type(op_type) == op_type ? 0 : 1;
777

778
        const Pipeline* pipeline = 0;
779
        if (A2.elempack == 1 && top_blob.elempack == 1)
780
        {
781
            pipeline = pipeline_binaryop_broadcast[ri];
782
        }
783
        if (A2.elempack == 4 && top_blob.elempack == 4)
784
        {
785
            pipeline = pipeline_binaryop_broadcast_pack4[ri];
786
        }
787
        if (A2.elempack == 1 && top_blob.elempack == 4)
788
        {
789
            pipeline = pipeline_binaryop_broadcast_pack1to4[ri];
790
        }
791
        if (A2.elempack == 8 && top_blob.elempack == 8)
792
        {
793
            pipeline = pipeline_binaryop_broadcast_pack8[ri];
794
        }
795
        if (A2.elempack == 1 && top_blob.elempack == 8)
796
        {
797
            pipeline = pipeline_binaryop_broadcast_pack1to8[ri];
798
        }
799

800
        cmd.record_pipeline(pipeline, bindings, constants, top_blob);
801
    }
802
    else
803
    {
804
        VkImageMat B2;
805
        if (B.dims == 1 && ((A.dims == 2 && B.w * B.elempack != A.h * A.elempack) || ((A.dims == 3 || A.dims == 4) && B.w * B.elempack != A.c * A.elempack)))
806
        {
807
            vkdev->convert_packing(B, B2, 1, cmd, opt);
808
            B2.dims = top_blob.dims;
809
        }
810
        else
811
        {
812
            B2 = B;
813
        }
814

815
        std::vector<VkImageMat> bindings(3);
816
        bindings[0] = A;
817
        bindings[1] = B2;
818
        bindings[2] = top_blob;
819

820
        std::vector<vk_constant_type> constants(18);
821
        constants[0].i = A.dims;
822
        constants[1].i = A.w;
823
        constants[2].i = A.h;
824
        constants[3].i = A.d;
825
        constants[4].i = A.c;
826
        constants[5].i = 0; //A.cstep;
827
        constants[6].i = B2.dims;
828
        constants[7].i = B2.w;
829
        constants[8].i = B2.h;
830
        constants[9].i = B2.d;
831
        constants[10].i = B2.c;
832
        constants[11].i = 0; //B2.cstep;
833
        constants[12].i = top_blob.dims;
834
        constants[13].i = top_blob.w;
835
        constants[14].i = top_blob.h;
836
        constants[15].i = top_blob.d;
837
        constants[16].i = top_blob.c;
838
        constants[17].i = 0; //top_blob.cstep;
839

840
        const Pipeline* pipeline = 0;
841
        if (B2.elempack == 1 && top_blob.elempack == 1)
842
        {
843
            pipeline = pipeline_binaryop_broadcast[0];
844
        }
845
        if (B2.elempack == 4 && top_blob.elempack == 4)
846
        {
847
            pipeline = pipeline_binaryop_broadcast_pack4[0];
848
        }
849
        if (B2.elempack == 1 && top_blob.elempack == 4)
850
        {
851
            pipeline = pipeline_binaryop_broadcast_pack1to4[0];
852
        }
853
        if (B2.elempack == 8 && top_blob.elempack == 8)
854
        {
855
            pipeline = pipeline_binaryop_broadcast_pack8[0];
856
        }
857
        if (B2.elempack == 1 && top_blob.elempack == 8)
858
        {
859
            pipeline = pipeline_binaryop_broadcast_pack1to8[0];
860
        }
861

862
        cmd.record_pipeline(pipeline, bindings, constants, top_blob);
863
    }
864

865
    return 0;
866
}
867

868
int BinaryOp_vulkan::forward_inplace(VkImageMat& bottom_top_blob, VkCompute& cmd, const Option& /*opt*/) const
869
{
870
    int elempack = bottom_top_blob.elempack;
871

872
    std::vector<VkImageMat> bindings(3);
873
    bindings[0] = bottom_top_blob;
874
    bindings[1] = bottom_top_blob; // TODO use dummy buffer
875
    bindings[2] = bottom_top_blob;
876

877
    std::vector<vk_constant_type> constants(15);
878
    constants[10].i = bottom_top_blob.dims;
879
    constants[11].i = bottom_top_blob.w;
880
    constants[12].i = bottom_top_blob.h * bottom_top_blob.d;
881
    constants[13].i = bottom_top_blob.c;
882
    constants[14].i = 0; //bottom_top_blob.cstep;
883

884
    const Pipeline* pipeline = elempack == 8 ? pipeline_binaryop_pack8
885
                               : elempack == 4 ? pipeline_binaryop_pack4
886
                               : pipeline_binaryop;
887

888
    cmd.record_pipeline(pipeline, bindings, constants, bottom_top_blob);
889

890
    return 0;
891
}
892

893
} // namespace ncnn
894