utils/update_metadata.py - main - githubmirror/transformers

utils/update_metadata.py
389 строк16 KB
Raushan Turganbay
Prevent auto-class from being modified for all models (#46844)
24 июн 2026, 21:15
24 июн 2026, 21:15c21da1b
# Copyright 2021 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Utility that updates the metadata of the Transformers library in the repository `huggingface/transformers-metadata`.

Usage for an update (as used by the GitHub action `update_metadata`):

```bash
python utils/update_metadata.py --token <token> --commit_sha <commit_sha>
```

Usage to check all pipelines are properly defined in the constant `PIPELINE_TAGS_AND_AUTO_MODELS` of this script, so
that new pipelines are properly added as metadata (as used in `make check-repo`):

```bash
python utils/update_metadata.py --check-only
```
"""

import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.models.auto.modeling_auto import MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES
from transformers.utils import direct_transformers_import


CHECKER_CONFIG = {
    "name": "update_metadata",
    "label": "Model metadata",
    # Approximate: imports the transformers module and inspects pipeline/auto mappings
    # at runtime. Does not iterate over files matching these globs directly.
    "cache_globs": ["src/transformers/models/**/*.py", "docs/**/*.md"],
    "check_args": ["--check-only"],
    # No safe local "fix" mode: running without `--check-only` pushes to the
    # `huggingface/transformers-metadata` Hub dataset (requires an auth token).
    # `fix_args=None` makes `make fix-repo` skip this checker, like other check-only ones.
    "fix_args": None,
}

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
TRANSFORMERS_PATH = "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match model names
_re_pt_models = re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration|ForRetrieval)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
PIPELINE_TAGS_AND_AUTO_MODELS = [
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("image-feature-extraction", "MODEL_FOR_IMAGE_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("any-to-any", "MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES", "AutoModelForMultimodalLM"),
    ("image-text-to-text", "MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES", "AutoModelForImageTextToText"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
    ("text-to-audio", "MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES", "AutoModelForTextToSpectrogram"),
    ("text-to-audio", "MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES", "AutoModelForTextToWaveform"),
    ("keypoint-matching", "MODEL_FOR_KEYPOINT_MATCHING_MAPPING_NAMES", "AutoModelForKeypointMatching"),
]


def camel_case_split(identifier: str) -> list[str]:
    """
    Split a camel-cased name into words.

    Args:
        identifier (`str`): The camel-cased name to parse.

    Returns:
        `List[str]`: The list of words in the identifier (as separated by capital letters).

    Example:

    ```py
    >>> camel_case_split("CamelCasedClass")
    ["Camel", "Cased", "Class"]
    ```
    """
    # Regex thanks to https://stackoverflow.com/questions/29916065/how-to-do-camelcase-split-in-python
    matches = re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier)
    return [m.group(0) for m in matches]


def get_frameworks_table() -> pd.DataFrame:
    """
    Generates a dataframe containing the supported auto classes for each model type, using the content of the auto
    modules.
    """
    # Dictionary model names to config.
    config_mapping_names = transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
    model_prefix_to_model_type = {
        config.replace("Config", ""): model_type for model_type, config in config_mapping_names.items()
    }

    pt_models = collections.defaultdict(bool)

    # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
    for attr_name in dir(transformers_module):
        lookup_dict = None
        if _re_pt_models.match(attr_name) is not None:
            lookup_dict = pt_models
            attr_name = _re_pt_models.match(attr_name).groups()[0]

        if lookup_dict is not None:
            while len(attr_name) > 0:
                if attr_name in model_prefix_to_model_type:
                    lookup_dict[model_prefix_to_model_type[attr_name]] = True
                    break
                # Try again after removing the last word in the name
                attr_name = "".join(camel_case_split(attr_name)[:-1])

    all_models = set(pt_models.keys())
    all_models = list(all_models)
    all_models.sort()

    data = {"model_type": all_models}
    data["pytorch"] = [pt_models[t] for t in all_models]

    # Now let's find the right processing class for each model. In order we check if there is a Processor, then a
    # Tokenizer, then a FeatureExtractor, then an ImageProcessor
    processors = {}
    for t in all_models:
        if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
            processors[t] = "AutoProcessor"
        elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
            processors[t] = "AutoTokenizer"
        elif t in transformers_module.models.auto.image_processing_auto.IMAGE_PROCESSOR_MAPPING_NAMES:
            processors[t] = "AutoImageProcessor"
        elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
            processors[t] = "AutoFeatureExtractor"
        else:
            # Default to AutoTokenizer if a model has nothing, for backward compatibility.
            processors[t] = "AutoTokenizer"

    data["processor"] = [processors[t] for t in all_models]

    return pd.DataFrame(data)


def update_pipeline_and_auto_class_table(table: dict[str, tuple[str, str]]) -> dict[str, tuple[str, str]]:
    """
    Update the table mapping models to pipelines and auto classes without removing old keys if they don't exist anymore.

    Args:
        table (`Dict[str, Tuple[str, str]]`):
            The existing table mapping model names to a tuple containing the pipeline tag and the auto-class name with
            which they should be used.

    Returns:
        `Dict[str, Tuple[str, str]]`: The updated table in the same format.
    """
    module = transformers_module.models.auto.modeling_auto
    for pipeline_tag, model_mapping, cls in PIPELINE_TAGS_AND_AUTO_MODELS:
        if not hasattr(module, model_mapping):
            continue

        # Iterate over all model_names for given mapping
        for names in getattr(module, model_mapping).values():
            if isinstance(names, str):
                names = [names]

            for name in names:
                # For multimodal LLMs, keep the fine-grained pipeline tag but use a generic `AutoClass`
                if name in MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES.values():
                    table[name] = (pipeline_tag, "AutoModelForMultimodalLM")
                else:
                    # Add pipeline tag and auto model class for those models
                    table[name] = (pipeline_tag, cls)

    return table


def update_metadata(token: str, commit_sha: str):
    """
    Update the metadata for the Transformers repo in `huggingface/transformers-metadata`.

    Args:
        token (`str`): A valid token giving write access to `huggingface/transformers-metadata`.
        commit_sha (`str`): The commit SHA on Transformers corresponding to this update.
    """
    frameworks_table = get_frameworks_table()
    frameworks_dataset = Dataset.from_pandas(frameworks_table)

    resolved_tags_file = hf_hub_download(
        "huggingface/transformers-metadata", "pipeline_tags.json", repo_type="dataset", token=token
    )
    tags_dataset = Dataset.from_json(resolved_tags_file)
    table = {
        tags_dataset[i]["model_class"]: (tags_dataset[i]["pipeline_tag"], tags_dataset[i]["auto_class"])
        for i in range(len(tags_dataset))
    }
    table = update_pipeline_and_auto_class_table(table)

    # Sort the model classes to avoid some nondeterministic updates to create false update commits.
    model_classes = sorted(table.keys())
    tags_table = pd.DataFrame(
        {
            "model_class": model_classes,
            "pipeline_tag": [table[m][0] for m in model_classes],
            "auto_class": [table[m][1] for m in model_classes],
        }
    )
    tags_dataset = Dataset.from_pandas(tags_table)

    hub_frameworks_json = hf_hub_download(
        repo_id="huggingface/transformers-metadata",
        filename="frameworks.json",
        repo_type="dataset",
        token=token,
    )
    with open(hub_frameworks_json) as f:
        hub_frameworks_json = f.read()

    hub_pipeline_tags_json = hf_hub_download(
        repo_id="huggingface/transformers-metadata",
        filename="pipeline_tags.json",
        repo_type="dataset",
        token=token,
    )
    with open(hub_pipeline_tags_json) as f:
        hub_pipeline_tags_json = f.read()

    with tempfile.TemporaryDirectory() as tmp_dir:
        frameworks_dataset.to_json(os.path.join(tmp_dir, "frameworks.json"))
        tags_dataset.to_json(os.path.join(tmp_dir, "pipeline_tags.json"))

        with open(os.path.join(tmp_dir, "frameworks.json")) as f:
            frameworks_json = f.read()
        with open(os.path.join(tmp_dir, "pipeline_tags.json")) as f:
            pipeline_tags_json = f.read()

        frameworks_equal = hub_frameworks_json == frameworks_json
        hub_pipeline_tags_equal = hub_pipeline_tags_json == pipeline_tags_json

        if frameworks_equal and hub_pipeline_tags_equal:
            print("No updates on the Hub, not pushing the metadata files.")
            return

        if commit_sha is not None:
            commit_message = (
                f"Update with commit {commit_sha}\n\nSee: "
                f"https://github.com/huggingface/transformers/commit/{commit_sha}"
            )
        else:
            commit_message = "Update"

        upload_folder(
            repo_id="huggingface/transformers-metadata",
            folder_path=tmp_dir,
            repo_type="dataset",
            token=token,
            commit_message=commit_message,
        )


def check_pipeline_tags():
    """
    Check all pipeline tags are properly defined in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant of this script.
    """
    in_table = {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
    pipeline_tasks = transformers_module.pipelines.SUPPORTED_TASKS
    missing = []
    for key in pipeline_tasks:
        if key not in in_table:
            model = pipeline_tasks[key]["pt"]
            if isinstance(model, (list, tuple)):
                model = model[0]
            model = model.__name__
            if model not in in_table.values():
                missing.append(key)

    if len(missing) > 0:
        msg = ", ".join(missing)
        raise ValueError(
            "The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside "
            f"`utils/update_metadata.py`: {msg}. Please add them!"
        )


def check_multimodal_auto_class_assignment():
    """
    Check that non-multimodal models are never assigned ``AutoModelForMultimodalLM`` as their auto
    class by ``update_pipeline_and_auto_class_table``.

    The bug this guards against: the old code mutated the ``cls`` loop variable when a multimodal
    model was encountered, causing subsequent non-multimodal models processed in the same outer
    loop iteration to inherit ``"AutoModelForMultimodalLM"`` instead of their correct auto class.
    """
    table = update_pipeline_and_auto_class_table({})
    multimodal_names = set(MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES.values())
    incorrect = [
        name
        for name, (_, auto_class) in table.items()
        if auto_class == "AutoModelForMultimodalLM" and name not in multimodal_names
    ]
    if incorrect:
        raise ValueError(
            "The following non-multimodal models were incorrectly assigned `AutoModelForMultimodalLM` "
            f"by `update_pipeline_and_auto_class_table`: {', '.join(sorted(incorrect))}. "
            "This is likely caused by mutating the `cls` loop variable inside the function."
        )


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
    parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
    parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
    args = parser.parse_args()

    if args.check_only:
        check_pipeline_tags()
        check_multimodal_auto_class_assignment()
    else:
        update_metadata(args.token, args.commit_sha)