ncnn

1
// Tencent is pleased to support the open source community by making ncnn available.
2
//
3
// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
4
//
5
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6
// in compliance with the License. You may obtain a copy of the License at
7
//
8
// https://opensource.org/licenses/BSD-3-Clause
9
//
10
// Unless required by applicable law or agreed to in writing, software distributed
11
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
13
// specific language governing permissions and limitations under the License.
14

15
#include "priorbox_vulkan.h"
16

17
#include "layer_shader_type.h"
18
#include "platform.h"
19

20
namespace ncnn {
21

22
PriorBox_vulkan::PriorBox_vulkan()
23
{
24
    support_vulkan = true;
25

26
    pipeline_priorbox = 0;
27
    pipeline_priorbox_mxnet = 0;
28
}
29

30
int PriorBox_vulkan::create_pipeline(const Option& opt)
31
{
32
    const Mat& shape = bottom_shapes.empty() ? Mat() : bottom_shapes[0];
33

34
    int elempack = 1;
35
    if (shape.dims == 1) elempack = opt.use_shader_pack8 && shape.w % 8 == 0 ? 8 : shape.w % 4 == 0 ? 4 : 1;
36
    if (shape.dims == 2) elempack = opt.use_shader_pack8 && shape.h % 8 == 0 ? 8 : shape.h % 4 == 0 ? 4 : 1;
37
    if (shape.dims == 3) elempack = opt.use_shader_pack8 && shape.c % 8 == 0 ? 8 : shape.c % 4 == 0 ? 4 : 1;
38

39
    size_t elemsize;
40
    if (opt.use_fp16_storage)
41
    {
42
        elemsize = elempack * 2u;
43
    }
44
    else if (opt.use_fp16_packed)
45
    {
46
        elemsize = elempack == 1 ? 4u : elempack * 2u;
47
    }
48
    else
49
    {
50
        elemsize = elempack * 4u;
51
    }
52

53
    Mat shape_packed;
54
    if (shape.dims == 1) shape_packed = Mat(shape.w / elempack, (void*)0, elemsize, elempack);
55
    if (shape.dims == 2) shape_packed = Mat(shape.w, shape.h / elempack, (void*)0, elemsize, elempack);
56
    if (shape.dims == 3) shape_packed = Mat(shape.w, shape.h, shape.c / elempack, (void*)0, elemsize, elempack);
57

58
    // caffe style
59
    {
60
        int num_min_size = min_sizes.w;
61
        int num_max_size = max_sizes.w;
62
        int num_aspect_ratio = aspect_ratios.w;
63

64
        int num_prior = num_min_size * num_aspect_ratio + num_min_size + num_max_size;
65
        if (flip)
66
            num_prior += num_min_size * num_aspect_ratio;
67

68
        std::vector<vk_specialization_type> specializations(11 + 2);
69
        specializations[0].i = flip;
70
        specializations[1].i = clip;
71
        specializations[2].f = offset;
72
        specializations[3].f = variances[0];
73
        specializations[4].f = variances[1];
74
        specializations[5].f = variances[2];
75
        specializations[6].f = variances[3];
76
        specializations[7].i = num_min_size;
77
        specializations[8].i = num_max_size;
78
        specializations[9].i = num_aspect_ratio;
79
        specializations[10].i = num_prior;
80
        specializations[11 + 0].i = shape_packed.w;
81
        specializations[11 + 1].i = shape_packed.h;
82

83
        pipeline_priorbox = new Pipeline(vkdev);
84
        pipeline_priorbox->set_optimal_local_size_xyz();
85
        pipeline_priorbox->create(LayerShaderType::priorbox, opt, specializations);
86
    }
87

88
    // mxnet style
89
    {
90
        int num_sizes = min_sizes.w;
91
        int num_ratios = aspect_ratios.w;
92

93
        int num_prior = num_sizes - 1 + num_ratios;
94

95
        std::vector<vk_specialization_type> specializations(5 + 2);
96
        specializations[0].i = clip;
97
        specializations[1].f = offset;
98
        specializations[2].i = num_sizes;
99
        specializations[3].i = num_ratios;
100
        specializations[4].i = num_prior;
101
        specializations[5 + 0].i = shape_packed.w;
102
        specializations[5 + 1].i = shape_packed.h;
103

104
        pipeline_priorbox_mxnet = new Pipeline(vkdev);
105
        pipeline_priorbox_mxnet->set_optimal_local_size_xyz();
106
        pipeline_priorbox_mxnet->create(LayerShaderType::priorbox_mxnet, opt, specializations);
107
    }
108

109
    return 0;
110
}
111

112
int PriorBox_vulkan::destroy_pipeline(const Option& /*opt*/)
113
{
114
    delete pipeline_priorbox;
115
    pipeline_priorbox = 0;
116

117
    delete pipeline_priorbox_mxnet;
118
    pipeline_priorbox_mxnet = 0;
119

120
    return 0;
121
}
122

123
int PriorBox_vulkan::upload_model(VkTransfer& cmd, const Option& opt)
124
{
125
    cmd.record_upload(min_sizes, min_sizes_gpu, opt);
126

127
    if (max_sizes.w > 0)
128
        cmd.record_upload(max_sizes, max_sizes_gpu, opt);
129

130
    cmd.record_upload(aspect_ratios, aspect_ratios_gpu, opt);
131

132
    if (opt.lightmode)
133
    {
134
        min_sizes.release();
135
        max_sizes.release();
136
        aspect_ratios.release();
137
    }
138

139
    return 0;
140
}
141

142
int PriorBox_vulkan::forward(const std::vector<VkMat>& bottom_blobs, std::vector<VkMat>& top_blobs, VkCompute& cmd, const Option& opt) const
143
{
144
    int w = bottom_blobs[0].w;
145
    int h = bottom_blobs[0].h;
146

147
    if (bottom_blobs.size() == 1 && image_width == -233 && image_height == -233 && max_sizes_gpu.empty())
148
    {
149
        // mxnet style _contrib_MultiBoxPrior
150
        float step_w = step_width;
151
        float step_h = step_height;
152
        if (step_w == -233)
153
            step_w = 1.f / (float)w;
154
        if (step_h == -233)
155
            step_h = 1.f / (float)h;
156

157
        int num_sizes = min_sizes_gpu.w;
158
        int num_ratios = aspect_ratios_gpu.w;
159

160
        int num_prior = num_sizes - 1 + num_ratios;
161

162
        int elempack = 4;
163

164
        size_t elemsize = elempack * 4u;
165
        if (opt.use_fp16_packed || opt.use_fp16_storage)
166
        {
167
            elemsize = elempack * 2u;
168
        }
169

170
        VkMat& top_blob = top_blobs[0];
171
        top_blob.create(4 * w * h * num_prior / elempack, elemsize, elempack, opt.blob_vkallocator);
172
        if (top_blob.empty())
173
            return -100;
174

175
        std::vector<VkMat> bindings(3);
176
        bindings[0] = top_blob;
177
        bindings[1] = min_sizes_gpu;
178
        bindings[2] = aspect_ratios_gpu;
179

180
        std::vector<vk_constant_type> constants(4);
181
        constants[0].i = w;
182
        constants[1].i = h;
183
        constants[2].f = step_w;
184
        constants[3].f = step_h;
185

186
        VkMat dispatcher;
187
        dispatcher.w = num_sizes;
188
        dispatcher.h = w;
189
        dispatcher.c = h;
190

191
        cmd.record_pipeline(pipeline_priorbox_mxnet, bindings, constants, dispatcher);
192

193
        return 0;
194
    }
195

196
    int image_w = image_width;
197
    int image_h = image_height;
198
    if (image_w == -233)
199
        image_w = bottom_blobs[1].w;
200
    if (image_h == -233)
201
        image_h = bottom_blobs[1].h;
202

203
    float step_w = step_width;
204
    float step_h = step_height;
205
    if (step_w == -233)
206
        step_w = (float)image_w / w;
207
    if (step_h == -233)
208
        step_h = (float)image_h / h;
209

210
    int num_min_size = min_sizes_gpu.w;
211
    int num_max_size = max_sizes_gpu.w;
212
    int num_aspect_ratio = aspect_ratios_gpu.w;
213

214
    int num_prior = num_min_size * num_aspect_ratio + num_min_size + num_max_size;
215
    if (flip)
216
        num_prior += num_min_size * num_aspect_ratio;
217

218
    size_t elemsize = 4u;
219
    if (opt.use_fp16_storage)
220
    {
221
        elemsize = 2u;
222
    }
223

224
    VkMat& top_blob = top_blobs[0];
225
    top_blob.create(4 * w * h * num_prior, 2, elemsize, 1, opt.blob_vkallocator);
226
    if (top_blob.empty())
227
        return -100;
228

229
    std::vector<VkMat> bindings(4);
230
    bindings[0] = top_blob;
231
    bindings[1] = min_sizes_gpu;
232
    bindings[2] = num_max_size > 0 ? max_sizes_gpu : min_sizes_gpu;
233
    bindings[3] = aspect_ratios_gpu;
234

235
    std::vector<vk_constant_type> constants(6);
236
    constants[0].i = w;
237
    constants[1].i = h;
238
    constants[2].f = image_w;
239
    constants[3].f = image_h;
240
    constants[4].f = step_w;
241
    constants[5].f = step_h;
242

243
    VkMat dispatcher;
244
    dispatcher.w = num_min_size;
245
    dispatcher.h = w;
246
    dispatcher.c = h;
247

248
    cmd.record_pipeline(pipeline_priorbox, bindings, constants, dispatcher);
249

250
    return 0;
251
}
252

253
} // namespace ncnn
254