/
rnekrasov
/
peft
GigaCode
beta
ОбзорЦентр заботыВойти

peft

Форк
0
Ветки: 10Коммиты: 876Теги: 16
peft
/tests
/
test_decoder_models.py 
304 строки · 14.3 Кб
1
# Copyright 2023-present the HuggingFace Inc. team.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14
import unittest
15
from unittest.mock import Mock, call, patch
16


17
import pytest
18
import torch
19
from parameterized import parameterized
20
from transformers import AutoModelForCausalLM, AutoTokenizer
21


22
from peft import AdaLoraConfig, PromptTuningConfig, PromptTuningInit, get_peft_model
23


24
from .testing_common import PeftCommonTester, PeftTestConfigManager
25


26


27
PEFT_DECODER_MODELS_TO_TEST = [
28
    "hf-internal-testing/tiny-random-OPTForCausalLM",
29
    "hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
30
    "hf-internal-testing/tiny-random-GPT2LMHeadModel",
31
    "hf-internal-testing/tiny-random-BloomForCausalLM",
32
    "hf-internal-testing/tiny-random-gpt_neo",
33
    "hf-internal-testing/tiny-random-GPTJForCausalLM",
34
    "hf-internal-testing/tiny-random-GPTBigCodeForCausalLM",
35
    "HuggingFaceM4/tiny-random-LlamaForCausalLM",
36
]
37


38
FULL_GRID = {
39
    "model_ids": PEFT_DECODER_MODELS_TO_TEST,
40
    "task_type": "CAUSAL_LM",
41
}
42


43


44
def skip_adalora_and_gpt2(test_list):
45
    return [test for test in test_list if not (("GPT2LMHeadModel" in test[1]) and (test[2] == AdaLoraConfig))]
46


47


48
class PeftDecoderModelTester(unittest.TestCase, PeftCommonTester):
49
    r"""
50
    Test if the PeftModel behaves as expected. This includes:
51
    - test if the model has the expected methods
52


53
    We use parametrized.expand for debugging purposes to test each model individually.
54
    """
55


56
    transformers_class = AutoModelForCausalLM
57


58
    def prepare_inputs_for_testing(self):
59
        input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device)
60
        attention_mask = torch.tensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device)
61


62
        input_dict = {
63
            "input_ids": input_ids,
64
            "attention_mask": attention_mask,
65
        }
66


67
        return input_dict
68


69
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
70
    def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs):
71
        self._test_model_attr(model_id, config_cls, config_kwargs)
72


73
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
74
    def test_adapter_name(self, test_name, model_id, config_cls, config_kwargs):
75
        self._test_adapter_name(model_id, config_cls, config_kwargs)
76


77
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
78
    def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs):
79
        self._test_prepare_for_training(model_id, config_cls, config_kwargs)
80


81
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
82
    def test_prompt_tuning_text_prepare_for_training(self, test_name, model_id, config_cls, config_kwargs):
83
        # Test that prompt tuning works with text init
84
        if config_cls != PromptTuningConfig:
85
            return pytest.skip(f"This test does not apply to {config_cls}")
86


87
        config_kwargs = config_kwargs.copy()
88
        config_kwargs["prompt_tuning_init"] = PromptTuningInit.TEXT
89
        config_kwargs["prompt_tuning_init_text"] = "This is a test prompt."
90
        config_kwargs["tokenizer_name_or_path"] = model_id
91
        self._test_prepare_for_training(model_id, config_cls, config_kwargs)
92


93
    def test_prompt_tuning_text_tokenizer_kwargs(self):
94
        # Allow users to pass additional arguments to Tokenizer.from_pretrained
95
        # Fix for #1032
96
        mock = Mock()
97
        orig_from_pretrained = AutoTokenizer.from_pretrained
98


99
        def mock_autotokenizer_from_pretrained(*args, **kwargs):
100
            mock(*args, **kwargs)
101
            return orig_from_pretrained(config.tokenizer_name_or_path)
102


103
        model_id = "hf-internal-testing/tiny-random-OPTForCausalLM"
104
        config = PromptTuningConfig(
105
            base_model_name_or_path=model_id,
106
            tokenizer_name_or_path=model_id,
107
            num_virtual_tokens=10,
108
            prompt_tuning_init=PromptTuningInit.TEXT,
109
            task_type="CAUSAL_LM",
110
            prompt_tuning_init_text="This is a test prompt.",
111
            tokenizer_kwargs={"trust_remote_code": True, "foo": "bar"},
112
        )
113
        model = self.transformers_class.from_pretrained(model_id).to(self.torch_device)
114
        with patch("transformers.AutoTokenizer.from_pretrained", mock_autotokenizer_from_pretrained):
115
            model = get_peft_model(model, config)
116


117
        expected_call = call(model_id, trust_remote_code=True, foo="bar")
118
        assert mock.call_args == expected_call
119


120
    def test_prompt_tuning_config_invalid_args(self):
121
        # Raise an error when tokenizer_kwargs is used with prompt_tuning_init!='TEXT', because this argument has no
122
        # function in that case
123
        model_id = "hf-internal-testing/tiny-random-OPTForCausalLM"
124
        with pytest.raises(ValueError, match="tokenizer_kwargs only valid when using prompt_tuning_init='TEXT'."):
125
            PromptTuningConfig(
126
                base_model_name_or_path=model_id,
127
                tokenizer_name_or_path=model_id,
128
                num_virtual_tokens=10,
129
                task_type="CAUSAL_LM",
130
                prompt_tuning_init_text="This is a test prompt.",
131
                prompt_tuning_init=PromptTuningInit.RANDOM,  # <= should not be used together with tokenizer_kwargs
132
                tokenizer_kwargs={"trust_remote_code": True, "foo": "bar"},
133
            )
134


135
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
136
    def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
137
        self._test_save_pretrained(model_id, config_cls, config_kwargs)
138


139
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
140
    def test_save_pretrained_pickle(self, test_name, model_id, config_cls, config_kwargs):
141
        self._test_save_pretrained(model_id, config_cls, config_kwargs, safe_serialization=False)
142


143
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
144
    def test_save_pretrained_selected_adapters(self, test_name, model_id, config_cls, config_kwargs):
145
        self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs)
146


147
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
148
    def test_save_pretrained_selected_adapters_pickle(self, test_name, model_id, config_cls, config_kwargs):
149
        self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs, safe_serialization=False)
150


151
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
152
    def test_from_pretrained_config_construction(self, test_name, model_id, config_cls, config_kwargs):
153
        self._test_from_pretrained_config_construction(model_id, config_cls, config_kwargs)
154


155
    @parameterized.expand(
156
        PeftTestConfigManager.get_grid_parameters(
157
            {
158
                "model_ids": PEFT_DECODER_MODELS_TO_TEST,
159
                "lora_kwargs": {"init_lora_weights": [False]},
160
                "ia3_kwargs": {"init_ia3_weights": [False]},
161
                "task_type": "CAUSAL_LM",
162
            },
163
        )
164
    )
165
    def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs):
166
        self._test_merge_layers(model_id, config_cls, config_kwargs)
167


168
    @parameterized.expand(
169
        PeftTestConfigManager.get_grid_parameters(
170
            {
171
                "model_ids": PEFT_DECODER_MODELS_TO_TEST,
172
                "lora_kwargs": {"init_lora_weights": [False]},
173
                "ia3_kwargs": {"init_ia3_weights": [False]},
174
                "task_type": "CAUSAL_LM",
175
            },
176
        )
177
    )
178
    def test_merge_layers_multi(self, test_name, model_id, config_cls, config_kwargs):
179
        self._test_merge_layers_multi(model_id, config_cls, config_kwargs)
180


181
    @parameterized.expand(
182
        PeftTestConfigManager.get_grid_parameters(
183
            {
184
                "model_ids": PEFT_DECODER_MODELS_TO_TEST,
185
                "lora_kwargs": {"init_lora_weights": [False]},
186
                "ia3_kwargs": {"init_ia3_weights": [False]},
187
                "task_type": "CAUSAL_LM",
188
            },
189
        )
190
    )
191
    def test_merge_layers_nan(self, test_name, model_id, config_cls, config_kwargs):
192
        self._test_merge_layers_nan(model_id, config_cls, config_kwargs)
193


194
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
195
    def test_generate(self, test_name, model_id, config_cls, config_kwargs):
196
        self._test_generate(model_id, config_cls, config_kwargs)
197


198
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
199
    def test_generate_pos_args(self, test_name, model_id, config_cls, config_kwargs):
200
        # positional args are supported for PeftModelForCausalLM
201
        self._test_generate_pos_args(model_id, config_cls, config_kwargs, raises_err=False)
202


203
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
204
    def test_merge_layers_fp16(self, test_name, model_id, config_cls, config_kwargs):
205
        self._test_merge_layers_fp16(model_id, config_cls, config_kwargs)
206


207
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
208
    def test_generate_half_prec(self, test_name, model_id, config_cls, config_kwargs):
209
        self._test_generate_half_prec(model_id, config_cls, config_kwargs)
210


211
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
212
    def test_prefix_tuning_half_prec_conversion(self, test_name, model_id, config_cls, config_kwargs):
213
        self._test_prefix_tuning_half_prec_conversion(model_id, config_cls, config_kwargs)
214


215
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
216
    def test_training_decoders(self, test_name, model_id, config_cls, config_kwargs):
217
        self._test_training(model_id, config_cls, config_kwargs)
218


219
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
220
    def test_training_decoders_layer_indexing(self, test_name, model_id, config_cls, config_kwargs):
221
        self._test_training_layer_indexing(model_id, config_cls, config_kwargs)
222


223
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
224
    def test_training_decoders_gradient_checkpointing(self, test_name, model_id, config_cls, config_kwargs):
225
        self._test_training_gradient_checkpointing(model_id, config_cls, config_kwargs)
226


227
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
228
    def test_inference_safetensors(self, test_name, model_id, config_cls, config_kwargs):
229
        self._test_inference_safetensors(model_id, config_cls, config_kwargs)
230


231
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
232
    def test_peft_model_device_map(self, test_name, model_id, config_cls, config_kwargs):
233
        self._test_peft_model_device_map(model_id, config_cls, config_kwargs)
234


235
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
236
    def test_delete_adapter(self, test_name, model_id, config_cls, config_kwargs):
237
        self._test_delete_adapter(model_id, config_cls, config_kwargs)
238


239
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
240
    def test_delete_inactive_adapter(self, test_name, model_id, config_cls, config_kwargs):
241
        self._test_delete_inactive_adapter(model_id, config_cls, config_kwargs)
242


243
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
244
    def test_adding_multiple_adapters_with_bias_raises(self, test_name, model_id, config_cls, config_kwargs):
245
        self._test_adding_multiple_adapters_with_bias_raises(model_id, config_cls, config_kwargs)
246


247
    @parameterized.expand(
248
        PeftTestConfigManager.get_grid_parameters(
249
            {
250
                "model_ids": PEFT_DECODER_MODELS_TO_TEST,
251
                "lora_kwargs": {"init_lora_weights": [False]},
252
                "adalora_kwargs": {"init_lora_weights": [False]},
253
                "ia3_kwargs": {"init_ia3_weights": [False]},
254
                "task_type": "CAUSAL_LM",
255
            },
256
            filter_params_func=skip_adalora_and_gpt2,
257
        )
258
    )
259
    def test_unload_adapter(self, test_name, model_id, config_cls, config_kwargs):
260
        self._test_unload_adapter(model_id, config_cls, config_kwargs)
261


262
    @parameterized.expand(
263
        PeftTestConfigManager.get_grid_parameters(
264
            {
265
                "model_ids": PEFT_DECODER_MODELS_TO_TEST,
266
                "lora_kwargs": {"init_lora_weights": [False]},
267
                "task_type": "CAUSAL_LM",
268
            },
269
        )
270
    )
271
    def test_weighted_combination_of_adapters(self, test_name, model_id, config_cls, config_kwargs):
272
        self._test_weighted_combination_of_adapters(model_id, config_cls, config_kwargs)
273


274
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
275
    def test_training_prompt_learning_tasks(self, test_name, model_id, config_cls, config_kwargs):
276
        self._test_training_prompt_learning_tasks(model_id, config_cls, config_kwargs)
277


278
    @parameterized.expand(
279
        PeftTestConfigManager.get_grid_parameters(
280
            {
281
                "model_ids": PEFT_DECODER_MODELS_TO_TEST,
282
                "lora_kwargs": {"init_lora_weights": [False]},
283
                "ia3_kwargs": {"init_ia3_weights": [False]},
284
                "adalora_kwargs": {"init_lora_weights": [False]},
285
                "task_type": "CAUSAL_LM",
286
            },
287
        )
288
    )
289
    def test_disable_adapter(self, test_name, model_id, config_cls, config_kwargs):
290
        self._test_disable_adapter(model_id, config_cls, config_kwargs)
291


292
    def test_generate_adalora_no_dropout(self):
293
        # test for issue #730
294
        model_id = "hf-internal-testing/tiny-random-OPTForCausalLM"
295
        config_kwargs = {
296
            "target_modules": None,
297
            "task_type": "CAUSAL_LM",
298
            "lora_dropout": 0.0,
299
        }
300
        self._test_generate(model_id, AdaLoraConfig, config_kwargs)
301


302
    @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
303
    def test_passing_input_embeds_works(self, test_name, model_id, config_cls, config_kwargs):
304
        self._test_passing_input_embeds_works(test_name, model_id, config_cls, config_kwargs)
305

Использование cookies

Мы используем файлы cookie в соответствии с Политикой конфиденциальности и Политикой использования cookies.

Нажимая кнопку «Принимаю», Вы даете АО «СберТех» согласие на обработку Ваших персональных данных в целях совершенствования нашего веб-сайта и Сервиса GitVerse, а также повышения удобства их использования.

Запретить использование cookies Вы можете самостоятельно в настройках Вашего браузера.