ncnn

1
// Tencent is pleased to support the open source community by making ncnn available.
2
//
3
// Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved.
4
//
5
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6
// in compliance with the License. You may obtain a copy of the License at
7
//
8
// https://opensource.org/licenses/BSD-3-Clause
9
//
10
// Unless required by applicable law or agreed to in writing, software distributed
11
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
13
// specific language governing permissions and limitations under the License.
14

15
#include "yolov3detectionoutput.h"
16

17
#include "layer_type.h"
18

19
#include <float.h>
20

21
namespace ncnn {
22

23
Yolov3DetectionOutput::Yolov3DetectionOutput()
24
{
25
    one_blob_only = false;
26
    support_inplace = false;
27

28
    //softmax = ncnn::create_layer_cpu(ncnn::LayerType::Softmax);
29

30
    // set param
31
    ncnn::ParamDict pd;
32
    pd.set(0, 0); // axis
33

34
    //softmax->load_param(pd);
35
}
36

37
Yolov3DetectionOutput::~Yolov3DetectionOutput()
38
{
39
    //delete softmax;
40
}
41

42
int Yolov3DetectionOutput::load_param(const ParamDict& pd)
43
{
44
    num_class = pd.get(0, 20);
45
    num_box = pd.get(1, 5);
46
    confidence_threshold = pd.get(2, 0.01f);
47
    nms_threshold = pd.get(3, 0.45f);
48
    biases = pd.get(4, Mat());
49
    mask = pd.get(5, Mat());
50
    anchors_scale = pd.get(6, Mat());
51
    return 0;
52
}
53

54
static inline float intersection_area(const Yolov3DetectionOutput::BBoxRect& a, const Yolov3DetectionOutput::BBoxRect& b)
55
{
56
    if (a.xmin > b.xmax || a.xmax < b.xmin || a.ymin > b.ymax || a.ymax < b.ymin)
57
    {
58
        // no intersection
59
        return 0.f;
60
    }
61

62
    float inter_width = std::min(a.xmax, b.xmax) - std::max(a.xmin, b.xmin);
63
    float inter_height = std::min(a.ymax, b.ymax) - std::max(a.ymin, b.ymin);
64

65
    return inter_width * inter_height;
66
}
67

68
void Yolov3DetectionOutput::qsort_descent_inplace(std::vector<BBoxRect>& datas, int left, int right) const
69
{
70
    int i = left;
71
    int j = right;
72
    float p = datas[(left + right) / 2].score;
73

74
    while (i <= j)
75
    {
76
        while (datas[i].score > p)
77
            i++;
78

79
        while (datas[j].score < p)
80
            j--;
81

82
        if (i <= j)
83
        {
84
            // swap
85
            std::swap(datas[i], datas[j]);
86

87
            i++;
88
            j--;
89
        }
90
    }
91

92
    if (left < j)
93
        qsort_descent_inplace(datas, left, j);
94

95
    if (i < right)
96
        qsort_descent_inplace(datas, i, right);
97
}
98

99
void Yolov3DetectionOutput::qsort_descent_inplace(std::vector<BBoxRect>& datas) const
100
{
101
    if (datas.empty())
102
        return;
103

104
    qsort_descent_inplace(datas, 0, static_cast<int>(datas.size() - 1));
105
}
106

107
void Yolov3DetectionOutput::nms_sorted_bboxes(std::vector<BBoxRect>& bboxes, std::vector<size_t>& picked, float nms_threshold) const
108
{
109
    picked.clear();
110

111
    const size_t n = bboxes.size();
112

113
    for (size_t i = 0; i < n; i++)
114
    {
115
        const BBoxRect& a = bboxes[i];
116

117
        int keep = 1;
118
        for (int j = 0; j < (int)picked.size(); j++)
119
        {
120
            const BBoxRect& b = bboxes[picked[j]];
121

122
            // intersection over union
123
            float inter_area = intersection_area(a, b);
124
            float union_area = a.area + b.area - inter_area;
125
            // float IoU = inter_area / union_area
126
            if (inter_area > nms_threshold * union_area)
127
            {
128
                keep = 0;
129
                break;
130
            }
131
        }
132

133
        if (keep)
134
            picked.push_back(i);
135
    }
136
}
137

138
static inline float sigmoid(float x)
139
{
140
    return 1.f / (1.f + expf(-x));
141
}
142

143
int Yolov3DetectionOutput::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
144
{
145
    // gather all box
146
    std::vector<BBoxRect> all_bbox_rects;
147

148
    for (size_t b = 0; b < bottom_blobs.size(); b++)
149
    {
150
        std::vector<std::vector<BBoxRect> > all_box_bbox_rects;
151
        all_box_bbox_rects.resize(num_box);
152
        const Mat& bottom_top_blobs = bottom_blobs[b];
153

154
        int w = bottom_top_blobs.w;
155
        int h = bottom_top_blobs.h;
156
        int channels = bottom_top_blobs.c;
157
        //printf("%d %d %d\n", w, h, channels);
158
        const int channels_per_box = channels / num_box;
159

160
        // anchor coord + box score + num_class
161
        if (channels_per_box != 4 + 1 + num_class)
162
            return -1;
163
        size_t mask_offset = b * num_box;
164
        int net_w = (int)(anchors_scale[b] * w);
165
        int net_h = (int)(anchors_scale[b] * h);
166
        //printf("%d %d\n", net_w, net_h);
167

168
        //printf("%d %d %d\n", w, h, channels);
169
        #pragma omp parallel for num_threads(opt.num_threads)
170
        for (int pp = 0; pp < num_box; pp++)
171
        {
172
            int p = pp * channels_per_box;
173
            int biases_index = static_cast<int>(mask[pp + mask_offset]);
174
            //printf("%d\n", biases_index);
175
            const float bias_w = biases[biases_index * 2];
176
            const float bias_h = biases[biases_index * 2 + 1];
177
            //printf("%f %f\n", bias_w, bias_h);
178
            const float* xptr = bottom_top_blobs.channel(p);
179
            const float* yptr = bottom_top_blobs.channel(p + 1);
180
            const float* wptr = bottom_top_blobs.channel(p + 2);
181
            const float* hptr = bottom_top_blobs.channel(p + 3);
182

183
            const float* box_score_ptr = bottom_top_blobs.channel(p + 4);
184

185
            // softmax class scores
186
            Mat scores = bottom_top_blobs.channel_range(p + 5, num_class);
187
            //softmax->forward_inplace(scores, opt);
188

189
            for (int i = 0; i < h; i++)
190
            {
191
                for (int j = 0; j < w; j++)
192
                {
193
                    // find class index with max class score
194
                    int class_index = 0;
195
                    float class_score = -FLT_MAX;
196
                    for (int q = 0; q < num_class; q++)
197
                    {
198
                        float score = scores.channel(q).row(i)[j];
199
                        if (score > class_score)
200
                        {
201
                            class_index = q;
202
                            class_score = score;
203
                        }
204
                    }
205

206
                    //sigmoid(box_score) * sigmoid(class_score)
207
                    float confidence = 1.f / ((1.f + expf(-box_score_ptr[0]) * (1.f + expf(-class_score))));
208
                    if (confidence >= confidence_threshold)
209
                    {
210
                        // region box
211
                        float bbox_cx = (j + sigmoid(xptr[0])) / w;
212
                        float bbox_cy = (i + sigmoid(yptr[0])) / h;
213
                        float bbox_w = expf(wptr[0]) * bias_w / net_w;
214
                        float bbox_h = expf(hptr[0]) * bias_h / net_h;
215

216
                        float bbox_xmin = bbox_cx - bbox_w * 0.5f;
217
                        float bbox_ymin = bbox_cy - bbox_h * 0.5f;
218
                        float bbox_xmax = bbox_cx + bbox_w * 0.5f;
219
                        float bbox_ymax = bbox_cy + bbox_h * 0.5f;
220

221
                        float area = bbox_w * bbox_h;
222

223
                        BBoxRect c = {confidence, bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, area, class_index};
224
                        all_box_bbox_rects[pp].push_back(c);
225
                    }
226

227
                    xptr++;
228
                    yptr++;
229
                    wptr++;
230
                    hptr++;
231

232
                    box_score_ptr++;
233
                }
234
            }
235
        }
236

237
        for (int i = 0; i < num_box; i++)
238
        {
239
            const std::vector<BBoxRect>& box_bbox_rects = all_box_bbox_rects[i];
240

241
            all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end());
242
        }
243
    }
244

245
    // global sort inplace
246
    qsort_descent_inplace(all_bbox_rects);
247

248
    // apply nms
249
    std::vector<size_t> picked;
250
    nms_sorted_bboxes(all_bbox_rects, picked, nms_threshold);
251

252
    // select
253
    std::vector<BBoxRect> bbox_rects;
254

255
    for (size_t i = 0; i < picked.size(); i++)
256
    {
257
        size_t z = picked[i];
258
        bbox_rects.push_back(all_bbox_rects[z]);
259
    }
260

261
    // fill result
262
    int num_detected = static_cast<int>(bbox_rects.size());
263
    if (num_detected == 0)
264
        return 0;
265

266
    Mat& top_blob = top_blobs[0];
267
    top_blob.create(6, num_detected, 4u, opt.blob_allocator);
268
    if (top_blob.empty())
269
        return -100;
270

271
    for (int i = 0; i < num_detected; i++)
272
    {
273
        const BBoxRect& r = bbox_rects[i];
274
        float score = r.score;
275
        float* outptr = top_blob.row(i);
276

277
        outptr[0] = r.label + 1.0f; // +1 for prepend background class
278
        outptr[1] = score;
279
        outptr[2] = r.xmin;
280
        outptr[3] = r.ymin;
281
        outptr[4] = r.xmax;
282
        outptr[5] = r.ymax;
283
    }
284

285
    return 0;
286
}
287

288
} // namespace ncnn
289