CSS-LM

1
# coding=utf-8
2
# Copyright 2020 The HuggingFace Inc. team.
3
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
4
#
5
# Licensed under the Apache License, Version 2.0 (the "License");
6
# you may not use this file except in compliance with the License.
7
# You may obtain a copy of the License at
8
#
9
#     http://www.apache.org/licenses/LICENSE-2.0
10
#
11
# Unless required by applicable law or agreed to in writing, software
12
# distributed under the License is distributed on an "AS IS" BASIS,
13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
# See the License for the specific language governing permissions and
15
# limitations under the License.
16

17
import copy
18
import logging
19

20
from .configuration_utils import PretrainedConfig
21

22

23
logger = logging.getLogger(__name__)
24

25

26
class EncoderDecoderConfig(PretrainedConfig):
27
    r"""
28
        :class:`~transformers.EncoderDecoderConfig` is the configuration class to store the configuration of a `EncoderDecoderModel`.
29

30
        It is used to instantiate an Encoder Decoder model according to the specified arguments, defining the encoder and decoder configs.
31
        Configuration objects inherit from  :class:`~transformers.PretrainedConfig`
32
        and can be used to control the model outputs.
33
        See the documentation for :class:`~transformers.PretrainedConfig` for more information.
34

35
        Args:
36
            kwargs (`optional`):
37
                Remaining dictionary of keyword arguments. Notably:
38
                    encoder (:class:`PretrainedConfig`, optional, defaults to `None`):
39
                        An instance of a configuration object that defines the encoder config.
40
                    decoder (:class:`PretrainedConfig`, optional, defaults to `None`):
41
                        An instance of a configuration object that defines the decoder config.
42

43
        Example::
44

45
            >>> from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel
46

47
            >>> # Initializing a BERT bert-base-uncased style configuration
48
            >>> config_encoder = BertConfig()
49
            >>> config_decoder = BertConfig()
50

51
            >>> config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
52

53
            >>> # Initializing a Bert2Bert model from the bert-base-uncased style configurations
54
            >>> model = EncoderDecoderModel(config=config)
55

56
            >>> # Accessing the model configuration
57
            >>> config_encoder = model.config.encoder
58
            >>> config_decoder  = model.config.decoder
59
    """
60
    model_type = "encoder_decoder"
61

62
    def __init__(self, **kwargs):
63
        super().__init__(**kwargs)
64
        assert (
65
            "encoder" in kwargs and "decoder" in kwargs
66
        ), "Config has to be initialized with encoder and decoder config"
67
        encoder_config = kwargs.pop("encoder")
68
        encoder_model_type = encoder_config.pop("model_type")
69
        decoder_config = kwargs.pop("decoder")
70
        decoder_model_type = decoder_config.pop("model_type")
71

72
        from .configuration_auto import AutoConfig
73

74
        self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
75
        self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)
76
        self.is_encoder_decoder = True
77

78
    @classmethod
79
    def from_encoder_decoder_configs(
80
        cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig
81
    ) -> PretrainedConfig:
82
        r"""
83
        Instantiate a :class:`~transformers.EncoderDecoderConfig` (or a derived class) from a pre-trained encoder model configuration and decoder model configuration.
84

85
        Returns:
86
            :class:`EncoderDecoderConfig`: An instance of a configuration object
87
        """
88
        logger.info("Set `config.is_decoder=True` for decoder_config")
89
        decoder_config.is_decoder = True
90

91
        return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict())
92

93
    def to_dict(self):
94
        """
95
        Serializes this instance to a Python dictionary. Override the default `to_dict()` from `PretrainedConfig`.
96

97
        Returns:
98
            :obj:`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
99
        """
100
        output = copy.deepcopy(self.__dict__)
101
        output["encoder"] = self.encoder.to_dict()
102
        output["decoder"] = self.decoder.to_dict()
103
        output["model_type"] = self.__class__.model_type
104
        return output
105