pytorch

1

2

3

4

5

6
from caffe2.python import core
7
from hypothesis import given, settings
8
import caffe2.python.hypothesis_test_util as hu
9
import caffe2.python.serialized_test.serialized_test_util as serial
10
import hypothesis.strategies as st
11
import numpy as np
12

13

14
# Reference implementation from detectron/lib/utils/boxes.py
15
def bbox_transform(boxes, deltas, weights=(1.0, 1.0, 1.0, 1.0)):
16
    """Forward transform that maps proposal boxes to predicted ground-truth
17
    boxes using bounding-box regression deltas. See bbox_transform_inv for a
18
    description of the weights argument.
19
    """
20
    if boxes.shape[0] == 0:
21
        return np.zeros((0, deltas.shape[1]), dtype=deltas.dtype)
22

23
    boxes = boxes.astype(deltas.dtype, copy=False)
24

25
    widths = boxes[:, 2] - boxes[:, 0] + 1.0
26
    heights = boxes[:, 3] - boxes[:, 1] + 1.0
27
    ctr_x = boxes[:, 0] + 0.5 * widths
28
    ctr_y = boxes[:, 1] + 0.5 * heights
29

30
    wx, wy, ww, wh = weights
31
    dx = deltas[:, 0::4] / wx
32
    dy = deltas[:, 1::4] / wy
33
    dw = deltas[:, 2::4] / ww
34
    dh = deltas[:, 3::4] / wh
35

36
    # Prevent sending too large values into np.exp()
37
    BBOX_XFORM_CLIP = np.log(1000. / 16.)
38
    dw = np.minimum(dw, BBOX_XFORM_CLIP)
39
    dh = np.minimum(dh, BBOX_XFORM_CLIP)
40

41
    pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis]
42
    pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis]
43
    pred_w = np.exp(dw) * widths[:, np.newaxis]
44
    pred_h = np.exp(dh) * heights[:, np.newaxis]
45

46
    pred_boxes = np.zeros(deltas.shape, dtype=deltas.dtype)
47
    # x1
48
    pred_boxes[:, 0::4] = pred_ctr_x - 0.5 * pred_w
49
    # y1
50
    pred_boxes[:, 1::4] = pred_ctr_y - 0.5 * pred_h
51
    # x2 (note: "- 1" is correct; don't be fooled by the asymmetry)
52
    pred_boxes[:, 2::4] = pred_ctr_x + 0.5 * pred_w - 1
53
    # y2 (note: "- 1" is correct; don't be fooled by the asymmetry)
54
    pred_boxes[:, 3::4] = pred_ctr_y + 0.5 * pred_h - 1
55

56
    return pred_boxes
57

58

59
# Reference implementation from detectron/lib/utils/boxes.py
60
def clip_tiled_boxes(boxes, im_shape):
61
    """Clip boxes to image boundaries. im_shape is [height, width] and boxes
62
    has shape (N, 4 * num_tiled_boxes)."""
63
    assert (
64
        boxes.shape[1] % 4 == 0
65
    ), "boxes.shape[1] is {:d}, but must be divisible by 4.".format(
66
        boxes.shape[1]
67
    )
68
    # x1 >= 0
69
    boxes[:, 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - 1), 0)
70
    # y1 >= 0
71
    boxes[:, 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - 1), 0)
72
    # x2 < im_shape[1]
73
    boxes[:, 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - 1), 0)
74
    # y2 < im_shape[0]
75
    boxes[:, 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - 1), 0)
76
    return boxes
77

78

79
def generate_rois(roi_counts, im_dims):
80
    assert len(roi_counts) == len(im_dims)
81
    all_rois = []
82
    for i, num_rois in enumerate(roi_counts):
83
        if num_rois == 0:
84
            continue
85
        # [batch_idx, x1, y1, x2, y2]
86
        rois = np.random.uniform(0, im_dims[i], size=(roi_counts[i], 5)).astype(
87
            np.float32
88
        )
89
        rois[:, 0] = i  # batch_idx
90
        # Swap (x1, x2) if x1 > x2
91
        rois[:, 1], rois[:, 3] = (
92
            np.minimum(rois[:, 1], rois[:, 3]),
93
            np.maximum(rois[:, 1], rois[:, 3]),
94
        )
95
        # Swap (y1, y2) if y1 > y2
96
        rois[:, 2], rois[:, 4] = (
97
            np.minimum(rois[:, 2], rois[:, 4]),
98
            np.maximum(rois[:, 2], rois[:, 4]),
99
        )
100
        all_rois.append(rois)
101
    if len(all_rois) > 0:
102
        return np.vstack(all_rois)
103
    return np.empty((0, 5)).astype(np.float32)
104

105

106
def bbox_transform_rotated(
107
    boxes,
108
    deltas,
109
    weights=(1.0, 1.0, 1.0, 1.0),
110
    angle_bound_on=True,
111
    angle_bound_lo=-90,
112
    angle_bound_hi=90,
113
):
114
    """
115
    Similar to bbox_transform but for rotated boxes with angle info.
116
    """
117
    if boxes.shape[0] == 0:
118
        return np.zeros((0, deltas.shape[1]), dtype=deltas.dtype)
119

120
    boxes = boxes.astype(deltas.dtype, copy=False)
121

122
    ctr_x = boxes[:, 0]
123
    ctr_y = boxes[:, 1]
124
    widths = boxes[:, 2]
125
    heights = boxes[:, 3]
126
    angles = boxes[:, 4]
127

128
    wx, wy, ww, wh = weights
129
    dx = deltas[:, 0::5] / wx
130
    dy = deltas[:, 1::5] / wy
131
    dw = deltas[:, 2::5] / ww
132
    dh = deltas[:, 3::5] / wh
133
    da = deltas[:, 4::5] * 180.0 / np.pi
134

135
    # Prevent sending too large values into np.exp()
136
    BBOX_XFORM_CLIP = np.log(1000. / 16.)
137
    dw = np.minimum(dw, BBOX_XFORM_CLIP)
138
    dh = np.minimum(dh, BBOX_XFORM_CLIP)
139

140
    pred_boxes = np.zeros(deltas.shape, dtype=deltas.dtype)
141
    pred_boxes[:, 0::5] = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis]
142
    pred_boxes[:, 1::5] = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis]
143
    pred_boxes[:, 2::5] = np.exp(dw) * widths[:, np.newaxis]
144
    pred_boxes[:, 3::5] = np.exp(dh) * heights[:, np.newaxis]
145

146
    pred_angle = da + angles[:, np.newaxis]
147
    if angle_bound_on:
148
        period = angle_bound_hi - angle_bound_lo
149
        assert period % 180 == 0
150
        pred_angle[np.where(pred_angle < angle_bound_lo)] += period
151
        pred_angle[np.where(pred_angle > angle_bound_hi)] -= period
152
    pred_boxes[:, 4::5] = pred_angle
153

154
    return pred_boxes
155

156

157
def clip_tiled_boxes_rotated(boxes, im_shape, angle_thresh=1.0):
158
    """
159
    Similar to clip_tiled_boxes but for rotated boxes with angle info.
160
    Only clips almost horizontal boxes within angle_thresh. The rest are
161
    left unchanged.
162
    """
163
    assert (
164
        boxes.shape[1] % 5 == 0
165
    ), "boxes.shape[1] is {:d}, but must be divisible by 5.".format(
166
        boxes.shape[1]
167
    )
168

169
    (H, W) = im_shape[:2]
170

171
    # Filter boxes that are almost upright within angle_thresh tolerance
172
    idx = np.where(np.abs(boxes[:, 4::5]) <= angle_thresh)
173
    idx5 = idx[1] * 5
174
    # convert to (x1, y1, x2, y2)
175
    x1 = boxes[idx[0], idx5] - (boxes[idx[0], idx5 + 2] - 1) / 2.0
176
    y1 = boxes[idx[0], idx5 + 1] - (boxes[idx[0], idx5 + 3] - 1) / 2.0
177
    x2 = boxes[idx[0], idx5] + (boxes[idx[0], idx5 + 2] - 1) / 2.0
178
    y2 = boxes[idx[0], idx5 + 1] + (boxes[idx[0], idx5 + 3] - 1) / 2.0
179
    # clip
180
    x1 = np.maximum(np.minimum(x1, W - 1), 0)
181
    y1 = np.maximum(np.minimum(y1, H - 1), 0)
182
    x2 = np.maximum(np.minimum(x2, W - 1), 0)
183
    y2 = np.maximum(np.minimum(y2, H - 1), 0)
184
    # convert back to (xc, yc, w, h)
185
    boxes[idx[0], idx5] = (x1 + x2) / 2.0
186
    boxes[idx[0], idx5 + 1] = (y1 + y2) / 2.0
187
    boxes[idx[0], idx5 + 2] = x2 - x1 + 1
188
    boxes[idx[0], idx5 + 3] = y2 - y1 + 1
189

190
    return boxes
191

192

193
def generate_rois_rotated(roi_counts, im_dims):
194
    rois = generate_rois(roi_counts, im_dims)
195
    # [batch_id, ctr_x, ctr_y, w, h, angle]
196
    rotated_rois = np.empty((rois.shape[0], 6)).astype(np.float32)
197
    rotated_rois[:, 0] = rois[:, 0]  # batch_id
198
    rotated_rois[:, 1] = (rois[:, 1] + rois[:, 3]) / 2.  # ctr_x = (x1 + x2) / 2
199
    rotated_rois[:, 2] = (rois[:, 2] + rois[:, 4]) / 2.  # ctr_y = (y1 + y2) / 2
200
    rotated_rois[:, 3] = rois[:, 3] - rois[:, 1] + 1.0  # w = x2 - x1 + 1
201
    rotated_rois[:, 4] = rois[:, 4] - rois[:, 2] + 1.0  # h = y2 - y1 + 1
202
    rotated_rois[:, 5] = np.random.uniform(-90.0, 90.0)  # angle in degrees
203
    return rotated_rois
204

205

206
class TestBBoxTransformOp(serial.SerializedTestCase):
207
    @given(
208
        num_rois=st.integers(1, 10),
209
        num_classes=st.integers(1, 10),
210
        im_dim=st.integers(100, 600),
211
        skip_batch_id=st.booleans(),
212
        rotated=st.booleans(),
213
        angle_bound_on=st.booleans(),
214
        clip_angle_thresh=st.sampled_from([-1.0, 1.0]),
215
        **hu.gcs_cpu_only
216
    )
217
    @settings(deadline=10000)
218
    def test_bbox_transform(
219
        self,
220
        num_rois,
221
        num_classes,
222
        im_dim,
223
        skip_batch_id,
224
        rotated,
225
        angle_bound_on,
226
        clip_angle_thresh,
227
        gc,
228
        dc,
229
    ):
230
        """
231
        Test with all rois belonging to a single image per run.
232
        """
233
        rois = (
234
            generate_rois_rotated([num_rois], [im_dim])
235
            if rotated
236
            else generate_rois([num_rois], [im_dim])
237
        )
238
        box_dim = 5 if rotated else 4
239
        if skip_batch_id:
240
            rois = rois[:, 1:]
241
        deltas = np.random.randn(num_rois, box_dim * num_classes).astype(np.float32)
242
        im_info = np.array([im_dim, im_dim, 1.0]).astype(np.float32).reshape(1, 3)
243

244
        def bbox_transform_ref(rois, deltas, im_info):
245
            boxes = rois if rois.shape[1] == box_dim else rois[:, 1:]
246
            im_shape = im_info[0, 0:2]
247
            if rotated:
248
                box_out = bbox_transform_rotated(
249
                    boxes, deltas, angle_bound_on=angle_bound_on
250
                )
251
                box_out = clip_tiled_boxes_rotated(
252
                    box_out, im_shape, angle_thresh=clip_angle_thresh
253
                )
254
            else:
255
                box_out = bbox_transform(boxes, deltas)
256
                box_out = clip_tiled_boxes(box_out, im_shape)
257
            return [box_out]
258

259
        op = core.CreateOperator(
260
            "BBoxTransform",
261
            ["rois", "deltas", "im_info"],
262
            ["box_out"],
263
            apply_scale=False,
264
            correct_transform_coords=True,
265
            rotated=rotated,
266
            angle_bound_on=angle_bound_on,
267
            clip_angle_thresh=clip_angle_thresh,
268
        )
269

270
        self.assertReferenceChecks(
271
            device_option=gc,
272
            op=op,
273
            inputs=[rois, deltas, im_info],
274
            reference=bbox_transform_ref,
275
        )
276

277
    @given(
278
        roi_counts=st.lists(st.integers(0, 5), min_size=1, max_size=10),
279
        num_classes=st.integers(1, 10),
280
        rotated=st.booleans(),
281
        angle_bound_on=st.booleans(),
282
        clip_angle_thresh=st.sampled_from([-1.0, 1.0]),
283
        **hu.gcs_cpu_only
284
    )
285
    @settings(deadline=10000)
286
    def test_bbox_transform_batch(
287
        self,
288
        roi_counts,
289
        num_classes,
290
        rotated,
291
        angle_bound_on,
292
        clip_angle_thresh,
293
        gc,
294
        dc,
295
    ):
296
        """
297
        Test with rois for multiple images in a batch
298
        """
299
        batch_size = len(roi_counts)
300
        total_rois = sum(roi_counts)
301
        im_dims = np.random.randint(100, 600, batch_size)
302
        rois = (
303
            generate_rois_rotated(roi_counts, im_dims)
304
            if rotated
305
            else generate_rois(roi_counts, im_dims)
306
        )
307
        box_dim = 5 if rotated else 4
308
        deltas = np.random.randn(total_rois, box_dim * num_classes).astype(np.float32)
309
        im_info = np.zeros((batch_size, 3)).astype(np.float32)
310
        im_info[:, 0] = im_dims
311
        im_info[:, 1] = im_dims
312
        im_info[:, 2] = 1.0
313

314
        def bbox_transform_ref(rois, deltas, im_info):
315
            box_out = []
316
            offset = 0
317
            for i, num_rois in enumerate(roi_counts):
318
                if num_rois == 0:
319
                    continue
320
                cur_boxes = rois[offset : offset + num_rois, 1:]
321
                cur_deltas = deltas[offset : offset + num_rois]
322
                im_shape = im_info[i, 0:2]
323
                if rotated:
324
                    cur_box_out = bbox_transform_rotated(
325
                        cur_boxes, cur_deltas, angle_bound_on=angle_bound_on
326
                    )
327
                    cur_box_out = clip_tiled_boxes_rotated(
328
                        cur_box_out, im_shape, angle_thresh=clip_angle_thresh
329
                    )
330
                else:
331
                    cur_box_out = bbox_transform(cur_boxes, cur_deltas)
332
                    cur_box_out = clip_tiled_boxes(cur_box_out, im_shape)
333
                box_out.append(cur_box_out)
334
                offset += num_rois
335

336
            if len(box_out) > 0:
337
                box_out = np.vstack(box_out)
338
            else:
339
                box_out = np.empty(deltas.shape).astype(np.float32)
340
            return [box_out, roi_counts]
341

342
        op = core.CreateOperator(
343
            "BBoxTransform",
344
            ["rois", "deltas", "im_info"],
345
            ["box_out", "roi_batch_splits"],
346
            apply_scale=False,
347
            correct_transform_coords=True,
348
            rotated=rotated,
349
            angle_bound_on=angle_bound_on,
350
            clip_angle_thresh=clip_angle_thresh,
351
        )
352

353
        self.assertReferenceChecks(
354
            device_option=gc,
355
            op=op,
356
            inputs=[rois, deltas, im_info],
357
            reference=bbox_transform_ref,
358
        )
359