transformers

1
# coding=utf-8
2
# Copyright 2020 The HuggingFace Team. All rights reserved.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15

16

17
import unittest
18

19
from transformers import is_torch_available
20
from transformers.testing_utils import require_torch, slow, torch_device
21

22
from ...generation.test_utils import GenerationTesterMixin
23
from ...test_configuration_common import ConfigTester
24
from ...test_modeling_common import ModelTesterMixin, ids_tensor
25
from ...test_pipeline_mixin import PipelineTesterMixin
26

27

28
if is_torch_available():
29
    import torch
30

31
    from transformers import (
32
        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
33
        OpenAIGPTConfig,
34
        OpenAIGPTDoubleHeadsModel,
35
        OpenAIGPTForSequenceClassification,
36
        OpenAIGPTLMHeadModel,
37
        OpenAIGPTModel,
38
    )
39

40

41
class OpenAIGPTModelTester:
42
    def __init__(
43
        self,
44
        parent,
45
        batch_size=13,
46
        seq_length=7,
47
        is_training=True,
48
        use_token_type_ids=True,
49
        use_labels=True,
50
        vocab_size=99,
51
        hidden_size=32,
52
        num_hidden_layers=2,
53
        num_attention_heads=4,
54
        intermediate_size=37,
55
        hidden_act="gelu",
56
        hidden_dropout_prob=0.1,
57
        attention_probs_dropout_prob=0.1,
58
        max_position_embeddings=512,
59
        type_vocab_size=16,
60
        type_sequence_label_size=2,
61
        initializer_range=0.02,
62
        num_labels=3,
63
        num_choices=4,
64
        scope=None,
65
    ):
66
        self.parent = parent
67
        self.batch_size = batch_size
68
        self.seq_length = seq_length
69
        self.is_training = is_training
70
        self.use_token_type_ids = use_token_type_ids
71
        self.use_labels = use_labels
72
        self.vocab_size = vocab_size
73
        self.hidden_size = hidden_size
74
        self.num_hidden_layers = num_hidden_layers
75
        self.num_attention_heads = num_attention_heads
76
        self.intermediate_size = intermediate_size
77
        self.hidden_act = hidden_act
78
        self.hidden_dropout_prob = hidden_dropout_prob
79
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
80
        self.max_position_embeddings = max_position_embeddings
81
        self.type_vocab_size = type_vocab_size
82
        self.type_sequence_label_size = type_sequence_label_size
83
        self.initializer_range = initializer_range
84
        self.num_labels = num_labels
85
        self.num_choices = num_choices
86
        self.scope = scope
87
        self.pad_token_id = self.vocab_size - 1
88

89
    def prepare_config_and_inputs(self):
90
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
91

92
        token_type_ids = None
93
        if self.use_token_type_ids:
94
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
95

96
        sequence_labels = None
97
        token_labels = None
98
        choice_labels = None
99
        if self.use_labels:
100
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
101
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
102
            choice_labels = ids_tensor([self.batch_size], self.num_choices)
103

104
        config = OpenAIGPTConfig(
105
            vocab_size=self.vocab_size,
106
            n_embd=self.hidden_size,
107
            n_layer=self.num_hidden_layers,
108
            n_head=self.num_attention_heads,
109
            # intermediate_size=self.intermediate_size,
110
            # hidden_act=self.hidden_act,
111
            # hidden_dropout_prob=self.hidden_dropout_prob,
112
            # attention_probs_dropout_prob=self.attention_probs_dropout_prob,
113
            n_positions=self.max_position_embeddings,
114
            # type_vocab_size=self.type_vocab_size,
115
            # initializer_range=self.initializer_range
116
            pad_token_id=self.pad_token_id,
117
        )
118

119
        head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2)
120

121
        return (
122
            config,
123
            input_ids,
124
            head_mask,
125
            token_type_ids,
126
            sequence_labels,
127
            token_labels,
128
            choice_labels,
129
        )
130

131
    def create_and_check_openai_gpt_model(self, config, input_ids, head_mask, token_type_ids, *args):
132
        model = OpenAIGPTModel(config=config)
133
        model.to(torch_device)
134
        model.eval()
135

136
        result = model(input_ids, token_type_ids=token_type_ids, head_mask=head_mask)
137
        result = model(input_ids, token_type_ids=token_type_ids)
138
        result = model(input_ids)
139

140
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
141

142
    def create_and_check_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args):
143
        model = OpenAIGPTLMHeadModel(config)
144
        model.to(torch_device)
145
        model.eval()
146

147
        result = model(input_ids, token_type_ids=token_type_ids, labels=input_ids)
148
        self.parent.assertEqual(result.loss.shape, ())
149
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
150

151
    def create_and_check_double_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args):
152
        model = OpenAIGPTDoubleHeadsModel(config)
153
        model.to(torch_device)
154
        model.eval()
155

156
        result = model(input_ids, token_type_ids=token_type_ids, labels=input_ids)
157
        self.parent.assertEqual(result.loss.shape, ())
158
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
159

160
    def create_and_check_openai_gpt_for_sequence_classification(
161
        self, config, input_ids, head_mask, token_type_ids, *args
162
    ):
163
        config.num_labels = self.num_labels
164
        model = OpenAIGPTForSequenceClassification(config)
165
        model.to(torch_device)
166
        model.eval()
167

168
        sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
169
        result = model(input_ids, token_type_ids=token_type_ids, labels=sequence_labels)
170
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
171

172
    def prepare_config_and_inputs_for_common(self):
173
        config_and_inputs = self.prepare_config_and_inputs()
174
        (
175
            config,
176
            input_ids,
177
            head_mask,
178
            token_type_ids,
179
            sequence_labels,
180
            token_labels,
181
            choice_labels,
182
        ) = config_and_inputs
183
        inputs_dict = {
184
            "input_ids": input_ids,
185
            "token_type_ids": token_type_ids,
186
            "head_mask": head_mask,
187
        }
188

189
        return config, inputs_dict
190

191

192
@require_torch
193
class OpenAIGPTModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
194
    all_model_classes = (
195
        (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
196
        if is_torch_available()
197
        else ()
198
    )
199
    all_generative_model_classes = (
200
        (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
201
    )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
202
    pipeline_model_mapping = (
203
        {
204
            "feature-extraction": OpenAIGPTModel,
205
            "text-classification": OpenAIGPTForSequenceClassification,
206
            "text-generation": OpenAIGPTLMHeadModel,
207
            "zero-shot": OpenAIGPTForSequenceClassification,
208
        }
209
        if is_torch_available()
210
        else {}
211
    )
212

213
    # TODO: Fix the failed tests
214
    def is_pipeline_test_to_skip(
215
        self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
216
    ):
217
        if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
218
            # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
219
            # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
220
            # tiny config could not be created.
221
            return True
222

223
        return False
224

225
    # special case for DoubleHeads model
226
    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
227
        inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)
228

229
        if return_labels:
230
            if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
231
                inputs_dict["labels"] = torch.zeros(
232
                    (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length),
233
                    dtype=torch.long,
234
                    device=torch_device,
235
                )
236
                inputs_dict["input_ids"] = inputs_dict["labels"]
237
                inputs_dict["token_type_ids"] = inputs_dict["labels"]
238
                inputs_dict["mc_token_ids"] = torch.zeros(
239
                    (self.model_tester.batch_size, self.model_tester.num_choices),
240
                    dtype=torch.long,
241
                    device=torch_device,
242
                )
243
                inputs_dict["mc_labels"] = torch.zeros(
244
                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
245
                )
246
        return inputs_dict
247

248
    def setUp(self):
249
        self.model_tester = OpenAIGPTModelTester(self)
250
        self.config_tester = ConfigTester(self, config_class=OpenAIGPTConfig, n_embd=37)
251

252
    def test_config(self):
253
        self.config_tester.run_common_tests()
254

255
    def test_openai_gpt_model(self):
256
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
257
        self.model_tester.create_and_check_openai_gpt_model(*config_and_inputs)
258

259
    def test_openai_gpt_lm_head_model(self):
260
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
261
        self.model_tester.create_and_check_lm_head_model(*config_and_inputs)
262

263
    def test_openai_gpt_double_lm_head_model(self):
264
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
265
        self.model_tester.create_and_check_double_lm_head_model(*config_and_inputs)
266

267
    def test_openai_gpt_classification_model(self):
268
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
269
        self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*config_and_inputs)
270

271
    @slow
272
    def test_model_from_pretrained(self):
273
        for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
274
            model = OpenAIGPTModel.from_pretrained(model_name)
275
            self.assertIsNotNone(model)
276

277

278
@require_torch
279
class OPENAIGPTModelLanguageGenerationTest(unittest.TestCase):
280
    @slow
281
    def test_lm_generate_openai_gpt(self):
282
        model = OpenAIGPTLMHeadModel.from_pretrained("openai-community/openai-gpt")
283
        model.to(torch_device)
284
        input_ids = torch.tensor([[481, 4735, 544]], dtype=torch.long, device=torch_device)  # the president is
285
        expected_output_ids = [
286
            481,
287
            4735,
288
            544,
289
            246,
290
            963,
291
            870,
292
            762,
293
            239,
294
            244,
295
            40477,
296
            244,
297
            249,
298
            719,
299
            881,
300
            487,
301
            544,
302
            240,
303
            244,
304
            603,
305
            481,
306
        ]  # the president is a very good man. " \n " i\'m sure he is, " said the
307

308
        output_ids = model.generate(input_ids, do_sample=False)
309
        self.assertListEqual(output_ids[0].tolist(), expected_output_ids)
310