/
githubmirror
/
transformers
Обзор
Документация
Войти
/
githubmirror
/
transformers
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
main
tests/models/auto/test_modeling_auto.py
599 строк
27 KB
Yih-Dar
Update a test to avoid writing to the default xet cache (#46250)
28 май 2026, 10:49
Не верифицирован
28 май 2026, 10:49
2d31b76
Код
Авторство
О чём код?
# Copyright 2020 The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import json import os import sys import tempfile import unittest from collections import OrderedDict from pathlib import Path import pytest import transformers from transformers import BertConfig, GPT2Model, is_torch_available from transformers.models.auto.configuration_auto import CONFIG_MAPPING from transformers.testing_utils import ( DUMMY_UNKNOWN_IDENTIFIER, RequestCounter, require_peft, require_torch, slow, ) from transformers.utils import ADAPTER_CONFIG_NAME from ..bert.test_modeling_bert import BertModelTester sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils")) from test_module.custom_configuration import CustomConfig # noqa E402 if is_torch_available(): import torch from test_module.custom_modeling import CustomModel from transformers import ( AutoBackbone, AutoConfig, AutoModel, AutoModelForCausalLM, AutoModelForMaskedLM, AutoModelForPreTraining, AutoModelForQuestionAnswering, AutoModelForSeq2SeqLM, AutoModelForSequenceClassification, AutoModelForTableQuestionAnswering, AutoModelForTokenClassification, BertForMaskedLM, BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BertForTokenClassification, BertModel, FunnelBaseModel, FunnelModel, GenerationMixin, GPT2Config, GPT2LMHeadModel, ResNetBackbone, T5Config, T5ForConditionalGeneration, TapasConfig, TapasForQuestionAnswering, TimmBackbone, ) from transformers.models.auto.modeling_auto import ( MODEL_FOR_CAUSAL_LM_MAPPING, MODEL_FOR_MASKED_LM_MAPPING, MODEL_FOR_PRETRAINING_MAPPING, MODEL_FOR_QUESTION_ANSWERING_MAPPING, MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, MODEL_MAPPING, ) @require_torch class AutoModelTest(unittest.TestCase): def setUp(self): transformers.dynamic_module_utils.TIME_OUT_REMOTE_CODE = 0 @slow def test_model_from_pretrained(self): model_name = "google-bert/bert-base-uncased" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) model = AutoModel.from_pretrained(model_name) model, loading_info = AutoModel.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, BertModel) self.assertEqual(len(loading_info["missing_keys"]), 0) # When using PyTorch checkpoint, the expected value is `8`. With `safetensors` checkpoint (if it is # installed), the expected value becomes `7`. EXPECTED_NUM_OF_UNEXPECTED_KEYS = 7 self.assertEqual(len(loading_info["unexpected_keys"]), EXPECTED_NUM_OF_UNEXPECTED_KEYS) self.assertEqual(len(loading_info["mismatched_keys"]), 0) self.assertEqual(len(loading_info["error_msgs"]), 0) @slow def test_model_for_pretraining_from_pretrained(self): model_name = "google-bert/bert-base-uncased" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) model = AutoModelForPreTraining.from_pretrained(model_name) model, loading_info = AutoModelForPreTraining.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, BertForPreTraining) # Only one value should not be initialized and in the missing keys. for value in loading_info.values(): self.assertEqual(len(value), 0) @slow def test_model_for_causal_lm(self): model_name = "openai-community/gpt2" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, GPT2Config) model = AutoModelForCausalLM.from_pretrained(model_name) model, loading_info = AutoModelForCausalLM.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, GPT2LMHeadModel) @slow def test_model_for_masked_lm(self): model_name = "google-bert/bert-base-uncased" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) model = AutoModelForMaskedLM.from_pretrained(model_name) model, loading_info = AutoModelForMaskedLM.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, BertForMaskedLM) @slow def test_model_for_encoder_decoder_lm(self): model_name = "google-t5/t5-base" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, T5Config) model = AutoModelForSeq2SeqLM.from_pretrained(model_name) model, loading_info = AutoModelForSeq2SeqLM.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, T5ForConditionalGeneration) @slow def test_sequence_classification_model_from_pretrained(self): model_name = "google-bert/bert-base-uncased" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) model = AutoModelForSequenceClassification.from_pretrained(model_name) model, loading_info = AutoModelForSequenceClassification.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, BertForSequenceClassification) @slow def test_question_answering_model_from_pretrained(self): model_name = "google-bert/bert-base-uncased" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) model = AutoModelForQuestionAnswering.from_pretrained(model_name) model, loading_info = AutoModelForQuestionAnswering.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, BertForQuestionAnswering) @slow def test_table_question_answering_model_from_pretrained(self): model_name = "google/tapas-base" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, TapasConfig) model = AutoModelForTableQuestionAnswering.from_pretrained(model_name) model, loading_info = AutoModelForTableQuestionAnswering.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, TapasForQuestionAnswering) @slow def test_token_classification_model_from_pretrained(self): model_name = "google-bert/bert-base-uncased" config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) model = AutoModelForTokenClassification.from_pretrained(model_name) model, loading_info = AutoModelForTokenClassification.from_pretrained(model_name, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, BertForTokenClassification) @slow def test_auto_backbone_timm_model_from_pretrained(self): # Configs can't be loaded for timm models model = AutoBackbone.from_pretrained("resnet18", use_timm_backbone=True) with pytest.raises(ValueError): # We can't pass output_loading_info=True as we're loading from timm AutoBackbone.from_pretrained("resnet18", use_timm_backbone=True, output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, TimmBackbone) # Check kwargs are correctly passed to the backbone model = AutoBackbone.from_pretrained("resnet18", use_timm_backbone=True, out_indices=(-2, -1)) self.assertEqual(model.out_indices, [-2, -1]) # Check out_features cannot be passed to Timm backbones with self.assertRaises(ValueError): _ = AutoBackbone.from_pretrained("resnet18", use_timm_backbone=True, out_features=["stage1"]) @slow def test_auto_backbone_from_pretrained(self): model = AutoBackbone.from_pretrained("microsoft/resnet-18") model, loading_info = AutoBackbone.from_pretrained("microsoft/resnet-18", output_loading_info=True) self.assertIsNotNone(model) self.assertIsInstance(model, ResNetBackbone) # Check kwargs are correctly passed to the backbone model = AutoBackbone.from_pretrained("microsoft/resnet-18", out_indices=[-2, -1]) self.assertEqual(model.out_indices, [-2, -1]) self.assertEqual(model.out_features, ["stage3", "stage4"]) model = AutoBackbone.from_pretrained("microsoft/resnet-18", out_features=["stage2", "stage4"]) self.assertEqual(model.out_indices, [2, 4]) self.assertEqual(model.out_features, ["stage2", "stage4"]) def test_from_pretrained_with_tuple_values(self): # For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel model = AutoModel.from_pretrained("sgugger/funnel-random-tiny") self.assertIsInstance(model, FunnelModel) config = copy.deepcopy(model.config) config.architectures = ["FunnelBaseModel"] model = AutoModel.from_config(config) self.assertIsInstance(model, FunnelBaseModel) with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained(tmp_dir) model = AutoModel.from_pretrained(tmp_dir) self.assertIsInstance(model, FunnelBaseModel) def test_from_pretrained_dynamic_model_local(self): try: AutoConfig.register("custom", CustomConfig) AutoModel.register(CustomConfig, CustomModel) config = CustomConfig(hidden_size=32) model = CustomModel(config) with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained(tmp_dir) new_model = AutoModel.from_pretrained(tmp_dir, trust_remote_code=True) for p1, p2 in zip(model.parameters(), new_model.parameters()): self.assertTrue(torch.equal(p1, p2)) finally: if "custom" in CONFIG_MAPPING._extra_content: del CONFIG_MAPPING._extra_content["custom"] if CustomConfig in MODEL_MAPPING._extra_content: del MODEL_MAPPING._extra_content[CustomConfig] def test_from_pretrained_dynamic_model_distant(self): # If remote code is not set, we will time out when asking whether to load the model. with self.assertRaises(ValueError): model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model") # If remote code is disabled, we can't load this config. with self.assertRaises(ValueError): model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=False) model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=True) self.assertEqual(model.__class__.__name__, "NewModel") # Test the dynamic module is loaded only once. reloaded_model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=True) self.assertIs(model.__class__, reloaded_model.__class__) # Test model can be reloaded. with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained(tmp_dir) reloaded_model = AutoModel.from_pretrained(tmp_dir, trust_remote_code=True) self.assertEqual(reloaded_model.__class__.__name__, "NewModel") for p1, p2 in zip(model.parameters(), reloaded_model.parameters()): self.assertTrue(torch.equal(p1, p2)) # Test the dynamic module is reloaded if we force it. reloaded_model = AutoModel.from_pretrained( "hf-internal-testing/test_dynamic_model", trust_remote_code=True, force_download=True ) self.assertIsNot(model.__class__, reloaded_model.__class__) # This one uses a relative import to a util file, this checks it is downloaded and used properly. model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model_with_util", trust_remote_code=True) self.assertEqual(model.__class__.__name__, "NewModel") # Test the dynamic module is loaded only once. reloaded_model = AutoModel.from_pretrained( "hf-internal-testing/test_dynamic_model_with_util", trust_remote_code=True ) self.assertIs(model.__class__, reloaded_model.__class__) # Test model can be reloaded. with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained(tmp_dir) reloaded_model = AutoModel.from_pretrained(tmp_dir, trust_remote_code=True) self.assertEqual(reloaded_model.__class__.__name__, "NewModel") for p1, p2 in zip(model.parameters(), reloaded_model.parameters()): self.assertTrue(torch.equal(p1, p2)) # Test the dynamic module is reloaded if we force it. reloaded_model = AutoModel.from_pretrained( "hf-internal-testing/test_dynamic_model_with_util", trust_remote_code=True, force_download=True ) self.assertIsNot(model.__class__, reloaded_model.__class__) def test_from_pretrained_dynamic_model_distant_with_ref(self): model = AutoModel.from_pretrained("hf-internal-testing/ref_to_test_dynamic_model", trust_remote_code=True) self.assertEqual(model.__class__.__name__, "NewModel") # Test model can be reloaded. with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained(tmp_dir) reloaded_model = AutoModel.from_pretrained(tmp_dir, trust_remote_code=True) self.assertEqual(reloaded_model.__class__.__name__, "NewModel") for p1, p2 in zip(model.parameters(), reloaded_model.parameters()): self.assertTrue(torch.equal(p1, p2)) # This one uses a relative import to a util file, this checks it is downloaded and used properly. model = AutoModel.from_pretrained( "hf-internal-testing/ref_to_test_dynamic_model_with_util", trust_remote_code=True ) self.assertEqual(model.__class__.__name__, "NewModel") # Test model can be reloaded. with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained(tmp_dir) reloaded_model = AutoModel.from_pretrained(tmp_dir, trust_remote_code=True) self.assertEqual(reloaded_model.__class__.__name__, "NewModel") for p1, p2 in zip(model.parameters(), reloaded_model.parameters()): self.assertTrue(torch.equal(p1, p2)) def test_from_pretrained_dynamic_model_with_period(self): # We used to have issues where repos with "." in the name would cause issues because the Python # import machinery would treat that as a directory separator, so we test that case # If remote code is not set, we will time out when asking whether to load the model. with self.assertRaises(ValueError): model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model_v1.0") # If remote code is disabled, we can't load this config. with self.assertRaises(ValueError): model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model_v1.0", trust_remote_code=False) model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model_v1.0", trust_remote_code=True) self.assertEqual(model.__class__.__name__, "NewModel") # Test that it works with a custom cache dir too with tempfile.TemporaryDirectory() as tmp_dir: with unittest.mock.patch.dict(os.environ, {"HF_XET_CACHE": tmp_dir}): model = AutoModel.from_pretrained( "hf-internal-testing/test_dynamic_model_v1.0", trust_remote_code=True, cache_dir=tmp_dir ) self.assertEqual(model.__class__.__name__, "NewModel") def test_new_model_registration(self): AutoConfig.register("custom", CustomConfig) auto_classes = [ AutoModel, AutoModelForCausalLM, AutoModelForMaskedLM, AutoModelForPreTraining, AutoModelForQuestionAnswering, AutoModelForSequenceClassification, AutoModelForTokenClassification, ] try: for auto_class in auto_classes: with self.subTest(auto_class.__name__): # Wrong config class will raise an error with self.assertRaises(ValueError): auto_class.register(BertConfig, CustomModel) auto_class.register(CustomConfig, CustomModel) # Trying to register something existing in the Transformers library will raise an error with self.assertRaises(ValueError): auto_class.register(BertConfig, BertModel) # Now that the config is registered, it can be used as any other config with the auto-API tiny_config = BertModelTester(self).get_config() config = CustomConfig(**tiny_config.to_dict()) model = auto_class.from_config(config) self.assertIsInstance(model, CustomModel) with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained(tmp_dir) new_model = auto_class.from_pretrained(tmp_dir) # The model is a CustomModel but from the new dynamically imported class. self.assertIsInstance(new_model, CustomModel) finally: if "custom" in CONFIG_MAPPING._extra_content: del CONFIG_MAPPING._extra_content["custom"] for mapping in ( MODEL_MAPPING, MODEL_FOR_PRETRAINING_MAPPING, MODEL_FOR_QUESTION_ANSWERING_MAPPING, MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, MODEL_FOR_CAUSAL_LM_MAPPING, MODEL_FOR_MASKED_LM_MAPPING, ): if CustomConfig in mapping._extra_content: del mapping._extra_content[CustomConfig] def test_from_pretrained_dynamic_model_conflict(self): class NewModelConfigLocal(BertConfig): model_type = "new-model" def __init__(self, **kwargs): super().__init__(**kwargs) class NewModel(BertModel): config_class = NewModelConfigLocal try: AutoConfig.register("new-model", NewModelConfigLocal) AutoModel.register(NewModelConfigLocal, NewModel) # If remote code is not set, the default is to use local model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model") self.assertEqual(model.config.__class__.__name__, "NewModelConfigLocal") # If remote code is disabled, we load the local one. model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=False) self.assertEqual(model.config.__class__.__name__, "NewModelConfigLocal") # If remote code is enabled but the user explicitly registered the local one, we load the local one. model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=True) self.assertEqual(model.config.__class__.__name__, "NewModelConfigLocal") # If remote code is enabled but local code originated from transformers, we load the remote one. NewModelConfigLocal.__module__ = "transformers.models.new_model.configuration_new_model" NewModel.__module__ = "transformers.models.new_model.modeling_new_model" model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=True) self.assertEqual(model.config.__class__.__name__, "NewModelConfig") finally: if "new-model" in CONFIG_MAPPING._extra_content: del CONFIG_MAPPING._extra_content["new-model"] if NewModelConfigLocal in MODEL_MAPPING._extra_content: del MODEL_MAPPING._extra_content[NewModelConfigLocal] def test_repo_not_found(self): with self.assertRaisesRegex( EnvironmentError, "bert-base is not a local folder and is not a valid model identifier" ): _ = AutoModel.from_pretrained("bert-base") def test_revision_not_found(self): with self.assertRaisesRegex( EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)" ): _ = AutoModel.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa") @unittest.skip("Failing on main") def test_cached_model_has_minimum_calls_to_head(self): # Make sure we have cached the model. _ = AutoModel.from_pretrained("hf-internal-testing/tiny-random-bert") with RequestCounter() as counter: _ = AutoModel.from_pretrained("hf-internal-testing/tiny-random-bert") self.assertEqual(counter["GET"], 0) self.assertEqual(counter["HEAD"], 1) self.assertEqual(counter.total_calls, 1) # With a sharded checkpoint _ = AutoModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded") with RequestCounter() as counter: _ = AutoModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded") self.assertEqual(counter["GET"], 0) self.assertEqual(counter["HEAD"], 1) self.assertEqual(counter.total_calls, 1) def test_attr_not_existing(self): from transformers.models.auto.auto_factory import _LazyAutoMapping _CONFIG_MAPPING_NAMES = OrderedDict([("bert", "BertConfig")]) _MODEL_MAPPING_NAMES = OrderedDict([("bert", "GhostModel")]) _MODEL_MAPPING = _LazyAutoMapping(_CONFIG_MAPPING_NAMES, _MODEL_MAPPING_NAMES) with pytest.raises(ValueError, match=r"Could not find GhostModel neither in .* nor in .*!"): _MODEL_MAPPING[BertConfig] _MODEL_MAPPING_NAMES = OrderedDict([("bert", "BertModel")]) _MODEL_MAPPING = _LazyAutoMapping(_CONFIG_MAPPING_NAMES, _MODEL_MAPPING_NAMES) self.assertEqual(_MODEL_MAPPING[BertConfig], BertModel) _MODEL_MAPPING_NAMES = OrderedDict([("bert", "GPT2Model")]) _MODEL_MAPPING = _LazyAutoMapping(_CONFIG_MAPPING_NAMES, _MODEL_MAPPING_NAMES) self.assertEqual(_MODEL_MAPPING[BertConfig], GPT2Model) def test_custom_model_patched_generation_inheritance(self): """ Tests that our inheritance patching for generate-compatible models works as expected. Without this feature, old Hub models lose the ability to call `generate`. """ model = AutoModelForCausalLM.from_pretrained( "hf-internal-testing/test_dynamic_model_generation", trust_remote_code=True ) self.assertTrue(model.__class__.__name__ == "NewModelForCausalLM") # It inherits from GenerationMixin. This means it can `generate`. Because `PreTrainedModel` is scheduled to # stop inheriting from `GenerationMixin` in v4.50, this check will fail if patching is not present. self.assertTrue(isinstance(model, GenerationMixin)) # More precisely, it directly inherits from GenerationMixin. This check would fail prior to v4.45 (inheritance # patching was added in v4.45) self.assertTrue("GenerationMixin" in str(model.__class__.__bases__)) @unittest.skip("@Cyril: add the post_init() on the hub repo") def test_model_with_dotted_name_and_relative_imports(self): """ Test for issue #40496: AutoModel.from_pretrained() doesn't work for models with '.' in their name when there's a relative import. Without the fix, this raises: ModuleNotFoundError: No module named 'transformers_modules.hf-internal-testing.remote_code_model_with_dots_v1' """ model_id = "hf-internal-testing/remote_code_model_with_dots_v1.0" model = AutoModel.from_pretrained(model_id, trust_remote_code=True) self.assertIsNotNone(model) @require_peft def test_adapter_path_not_overwritten_for_complete_model(self): """ Test for issue #43746: Only overwrite the pretrained_model_name_or_path if needed with adapter. This test ensures that when a model has an adapter config and the pretrained_model_name_or_path points to a model directory with both a base model and an embedded adapter, the path should NOT be overwritten with the hub model name embedded in the adapter's config. The bug was that the path was being unconditionally overwritten, which would cause incorrect behavior when loading models with adapters that are embedded within the same directory as the base model. """ peft_test_model = "peft-internal-testing/tiny-OPTForCausalLM-lora" transformers_test_model = "hf-internal-testing/tiny-random-OPTForCausalLM" # Create a temporary directory with a complete adapter model structure with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) # Save the model and adapter locally config = AutoConfig.from_pretrained(transformers_test_model) model = AutoModel.from_pretrained(transformers_test_model) adapter_model = AutoModel.from_pretrained(peft_test_model) config.save_pretrained(tmp_dir) model.save_pretrained(tmp_dir) adapter_model.save_pretrained(tmp_dir) # Overwrite the base_model_name_or_path to an invalid value that # would cause the load to fail later adapter_config_path = tmp_dir / ADAPTER_CONFIG_NAME with open(adapter_config_path, "r") as handle: adapter_config = json.load(handle) adapter_config["base_model_name_or_path"] = "some/model/that/does/not/exist" with open(adapter_config_path, "w") as handle: json.dump(adapter_config, handle) # Load from the saved path and make sure it actually loads despite # the invalid adapter config path AutoModel.from_pretrained(tmp_dir)