transformers

1
# coding=utf-8
2
# Copyright 2021 HuggingFace Inc.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15

16

17
import unittest
18

19
from datasets import load_dataset
20

21
from transformers.testing_utils import require_torch, require_vision
22
from transformers.utils import is_torch_available, is_vision_available
23

24
from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs
25

26

27
if is_torch_available():
28
    import torch
29

30
if is_vision_available():
31
    from PIL import Image
32

33
    from transformers import BeitImageProcessor
34

35

36
class BeitImageProcessingTester(unittest.TestCase):
37
    def __init__(
38
        self,
39
        parent,
40
        batch_size=7,
41
        num_channels=3,
42
        image_size=18,
43
        min_resolution=30,
44
        max_resolution=400,
45
        do_resize=True,
46
        size=None,
47
        do_center_crop=True,
48
        crop_size=None,
49
        do_normalize=True,
50
        image_mean=[0.5, 0.5, 0.5],
51
        image_std=[0.5, 0.5, 0.5],
52
        do_reduce_labels=False,
53
    ):
54
        size = size if size is not None else {"height": 20, "width": 20}
55
        crop_size = crop_size if crop_size is not None else {"height": 18, "width": 18}
56
        self.parent = parent
57
        self.batch_size = batch_size
58
        self.num_channels = num_channels
59
        self.image_size = image_size
60
        self.min_resolution = min_resolution
61
        self.max_resolution = max_resolution
62
        self.do_resize = do_resize
63
        self.size = size
64
        self.do_center_crop = do_center_crop
65
        self.crop_size = crop_size
66
        self.do_normalize = do_normalize
67
        self.image_mean = image_mean
68
        self.image_std = image_std
69
        self.do_reduce_labels = do_reduce_labels
70

71
    def prepare_image_processor_dict(self):
72
        return {
73
            "do_resize": self.do_resize,
74
            "size": self.size,
75
            "do_center_crop": self.do_center_crop,
76
            "crop_size": self.crop_size,
77
            "do_normalize": self.do_normalize,
78
            "image_mean": self.image_mean,
79
            "image_std": self.image_std,
80
            "do_reduce_labels": self.do_reduce_labels,
81
        }
82

83
    def expected_output_image_shape(self, images):
84
        return self.num_channels, self.crop_size["height"], self.crop_size["width"]
85

86
    def prepare_image_inputs(self, equal_resolution=False, numpify=False, torchify=False):
87
        return prepare_image_inputs(
88
            batch_size=self.batch_size,
89
            num_channels=self.num_channels,
90
            min_resolution=self.min_resolution,
91
            max_resolution=self.max_resolution,
92
            equal_resolution=equal_resolution,
93
            numpify=numpify,
94
            torchify=torchify,
95
        )
96

97

98
def prepare_semantic_single_inputs():
99
    dataset = load_dataset("hf-internal-testing/fixtures_ade20k", split="test")
100

101
    image = Image.open(dataset[0]["file"])
102
    map = Image.open(dataset[1]["file"])
103

104
    return image, map
105

106

107
def prepare_semantic_batch_inputs():
108
    ds = load_dataset("hf-internal-testing/fixtures_ade20k", split="test")
109

110
    image1 = Image.open(ds[0]["file"])
111
    map1 = Image.open(ds[1]["file"])
112
    image2 = Image.open(ds[2]["file"])
113
    map2 = Image.open(ds[3]["file"])
114

115
    return [image1, image2], [map1, map2]
116

117

118
@require_torch
119
@require_vision
120
class BeitImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
121
    image_processing_class = BeitImageProcessor if is_vision_available() else None
122

123
    def setUp(self):
124
        self.image_processor_tester = BeitImageProcessingTester(self)
125

126
    @property
127
    def image_processor_dict(self):
128
        return self.image_processor_tester.prepare_image_processor_dict()
129

130
    def test_image_processor_properties(self):
131
        image_processing = self.image_processing_class(**self.image_processor_dict)
132
        self.assertTrue(hasattr(image_processing, "do_resize"))
133
        self.assertTrue(hasattr(image_processing, "size"))
134
        self.assertTrue(hasattr(image_processing, "do_center_crop"))
135
        self.assertTrue(hasattr(image_processing, "center_crop"))
136
        self.assertTrue(hasattr(image_processing, "do_normalize"))
137
        self.assertTrue(hasattr(image_processing, "image_mean"))
138
        self.assertTrue(hasattr(image_processing, "image_std"))
139

140
    def test_image_processor_from_dict_with_kwargs(self):
141
        image_processor = self.image_processing_class.from_dict(self.image_processor_dict)
142
        self.assertEqual(image_processor.size, {"height": 20, "width": 20})
143
        self.assertEqual(image_processor.crop_size, {"height": 18, "width": 18})
144
        self.assertEqual(image_processor.do_reduce_labels, False)
145

146
        image_processor = self.image_processing_class.from_dict(
147
            self.image_processor_dict, size=42, crop_size=84, reduce_labels=True
148
        )
149
        self.assertEqual(image_processor.size, {"height": 42, "width": 42})
150
        self.assertEqual(image_processor.crop_size, {"height": 84, "width": 84})
151
        self.assertEqual(image_processor.do_reduce_labels, True)
152

153
    def test_call_segmentation_maps(self):
154
        # Initialize image_processing
155
        image_processing = self.image_processing_class(**self.image_processor_dict)
156
        # create random PyTorch tensors
157
        image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True)
158
        maps = []
159
        for image in image_inputs:
160
            self.assertIsInstance(image, torch.Tensor)
161
            maps.append(torch.zeros(image.shape[-2:]).long())
162

163
        # Test not batched input
164
        encoding = image_processing(image_inputs[0], maps[0], return_tensors="pt")
165
        self.assertEqual(
166
            encoding["pixel_values"].shape,
167
            (
168
                1,
169
                self.image_processor_tester.num_channels,
170
                self.image_processor_tester.crop_size["height"],
171
                self.image_processor_tester.crop_size["width"],
172
            ),
173
        )
174
        self.assertEqual(
175
            encoding["labels"].shape,
176
            (
177
                1,
178
                self.image_processor_tester.crop_size["height"],
179
                self.image_processor_tester.crop_size["width"],
180
            ),
181
        )
182
        self.assertEqual(encoding["labels"].dtype, torch.long)
183
        self.assertTrue(encoding["labels"].min().item() >= 0)
184
        self.assertTrue(encoding["labels"].max().item() <= 255)
185

186
        # Test batched
187
        encoding = image_processing(image_inputs, maps, return_tensors="pt")
188
        self.assertEqual(
189
            encoding["pixel_values"].shape,
190
            (
191
                self.image_processor_tester.batch_size,
192
                self.image_processor_tester.num_channels,
193
                self.image_processor_tester.crop_size["height"],
194
                self.image_processor_tester.crop_size["width"],
195
            ),
196
        )
197
        self.assertEqual(
198
            encoding["labels"].shape,
199
            (
200
                self.image_processor_tester.batch_size,
201
                self.image_processor_tester.crop_size["height"],
202
                self.image_processor_tester.crop_size["width"],
203
            ),
204
        )
205
        self.assertEqual(encoding["labels"].dtype, torch.long)
206
        self.assertTrue(encoding["labels"].min().item() >= 0)
207
        self.assertTrue(encoding["labels"].max().item() <= 255)
208

209
        # Test not batched input (PIL images)
210
        image, segmentation_map = prepare_semantic_single_inputs()
211

212
        encoding = image_processing(image, segmentation_map, return_tensors="pt")
213
        self.assertEqual(
214
            encoding["pixel_values"].shape,
215
            (
216
                1,
217
                self.image_processor_tester.num_channels,
218
                self.image_processor_tester.crop_size["height"],
219
                self.image_processor_tester.crop_size["width"],
220
            ),
221
        )
222
        self.assertEqual(
223
            encoding["labels"].shape,
224
            (
225
                1,
226
                self.image_processor_tester.crop_size["height"],
227
                self.image_processor_tester.crop_size["width"],
228
            ),
229
        )
230
        self.assertEqual(encoding["labels"].dtype, torch.long)
231
        self.assertTrue(encoding["labels"].min().item() >= 0)
232
        self.assertTrue(encoding["labels"].max().item() <= 255)
233

234
        # Test batched input (PIL images)
235
        images, segmentation_maps = prepare_semantic_batch_inputs()
236

237
        encoding = image_processing(images, segmentation_maps, return_tensors="pt")
238
        self.assertEqual(
239
            encoding["pixel_values"].shape,
240
            (
241
                2,
242
                self.image_processor_tester.num_channels,
243
                self.image_processor_tester.crop_size["height"],
244
                self.image_processor_tester.crop_size["width"],
245
            ),
246
        )
247
        self.assertEqual(
248
            encoding["labels"].shape,
249
            (
250
                2,
251
                self.image_processor_tester.crop_size["height"],
252
                self.image_processor_tester.crop_size["width"],
253
            ),
254
        )
255
        self.assertEqual(encoding["labels"].dtype, torch.long)
256
        self.assertTrue(encoding["labels"].min().item() >= 0)
257
        self.assertTrue(encoding["labels"].max().item() <= 255)
258

259
    def test_reduce_labels(self):
260
        # Initialize image_processing
261
        image_processing = self.image_processing_class(**self.image_processor_dict)
262

263
        # ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
264
        image, map = prepare_semantic_single_inputs()
265
        encoding = image_processing(image, map, return_tensors="pt")
266
        self.assertTrue(encoding["labels"].min().item() >= 0)
267
        self.assertTrue(encoding["labels"].max().item() <= 150)
268

269
        image_processing.do_reduce_labels = True
270
        encoding = image_processing(image, map, return_tensors="pt")
271
        self.assertTrue(encoding["labels"].min().item() >= 0)
272
        self.assertTrue(encoding["labels"].max().item() <= 255)
273