transformers

1
# coding=utf-8
2
# Copyright 2021 The HuggingFace Team All rights reserved.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15
"""
16
A subclass of `Trainer` specific to Question-Answering tasks
17
"""
18
import math
19
import time
20
from typing import Dict, List, Optional
21

22
from torch.utils.data import Dataset
23

24
from transformers import Seq2SeqTrainer, is_torch_tpu_available
25
from transformers.trainer_utils import PredictionOutput, speed_metrics
26

27

28
if is_torch_tpu_available(check_device=False):
29
    import torch_xla.core.xla_model as xm
30
    import torch_xla.debug.metrics as met
31

32

33
class QuestionAnsweringSeq2SeqTrainer(Seq2SeqTrainer):
34
    def __init__(self, *args, eval_examples=None, post_process_function=None, **kwargs):
35
        super().__init__(*args, **kwargs)
36
        self.eval_examples = eval_examples
37
        self.post_process_function = post_process_function
38

39
    # def evaluate(self, eval_dataset=None, eval_examples=None, ignore_keys=None, metric_key_prefix: str = "eval"):
40
    def evaluate(
41
        self,
42
        eval_dataset: Optional[Dataset] = None,
43
        eval_examples=None,
44
        ignore_keys: Optional[List[str]] = None,
45
        metric_key_prefix: str = "eval",
46
        **gen_kwargs,
47
    ) -> Dict[str, float]:
48
        gen_kwargs = gen_kwargs.copy()
49

50
        # Use legacy argument setting if a) the option is not explicitly passed; and b) the argument is set in the
51
        # training args
52
        if gen_kwargs.get("max_length") is None and self.args.generation_max_length is not None:
53
            gen_kwargs["max_length"] = self.args.generation_max_length
54
        if gen_kwargs.get("num_beams") is None and self.args.generation_num_beams is not None:
55
            gen_kwargs["num_beams"] = self.args.generation_num_beams
56
        self._gen_kwargs = gen_kwargs
57

58
        eval_dataset = self.eval_dataset if eval_dataset is None else eval_dataset
59
        eval_dataloader = self.get_eval_dataloader(eval_dataset)
60
        eval_examples = self.eval_examples if eval_examples is None else eval_examples
61

62
        # Temporarily disable metric computation, we will do it in the loop here.
63
        compute_metrics = self.compute_metrics
64
        self.compute_metrics = None
65
        start_time = time.time()
66
        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
67
        try:
68
            output = eval_loop(
69
                eval_dataloader,
70
                description="Evaluation",
71
                # No point gathering the predictions if there are no metrics, otherwise we defer to
72
                # self.args.prediction_loss_only
73
                prediction_loss_only=True if compute_metrics is None else None,
74
                ignore_keys=ignore_keys,
75
                metric_key_prefix=metric_key_prefix,
76
            )
77
        finally:
78
            self.compute_metrics = compute_metrics
79
        total_batch_size = self.args.eval_batch_size * self.args.world_size
80
        if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
81
            start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
82
        output.metrics.update(
83
            speed_metrics(
84
                metric_key_prefix,
85
                start_time,
86
                num_samples=output.num_samples,
87
                num_steps=math.ceil(output.num_samples / total_batch_size),
88
            )
89
        )
90

91
        if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
92
            # Only the main node write the results by default
93
            eval_preds = self.post_process_function(eval_examples, eval_dataset, output)
94
            metrics = self.compute_metrics(eval_preds)
95

96
            # Prefix all keys with metric_key_prefix + '_'
97
            for key in list(metrics.keys()):
98
                if not key.startswith(f"{metric_key_prefix}_"):
99
                    metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
100

101
            metrics.update(output.metrics)
102
        else:
103
            metrics = output.metrics
104

105
        if self.args.should_log:
106
            # Only the main node log the results by default
107
            self.log(metrics)
108

109
        if self.args.tpu_metrics_debug or self.args.debug:
110
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
111
            xm.master_print(met.metrics_report())
112

113
        self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)
114
        return metrics
115

116
    def predict(
117
        self, predict_dataset, predict_examples, ignore_keys=None, metric_key_prefix: str = "test", **gen_kwargs
118
    ):
119
        self._gen_kwargs = gen_kwargs.copy()
120

121
        predict_dataloader = self.get_test_dataloader(predict_dataset)
122

123
        # Temporarily disable metric computation, we will do it in the loop here.
124
        compute_metrics = self.compute_metrics
125
        self.compute_metrics = None
126
        start_time = time.time()
127
        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
128
        try:
129
            output = eval_loop(
130
                predict_dataloader,
131
                description="Prediction",
132
                # No point gathering the predictions if there are no metrics, otherwise we defer to
133
                # self.args.prediction_loss_only
134
                prediction_loss_only=True if compute_metrics is None else None,
135
                ignore_keys=ignore_keys,
136
                metric_key_prefix=metric_key_prefix,
137
            )
138
        finally:
139
            self.compute_metrics = compute_metrics
140

141
        total_batch_size = self.args.eval_batch_size * self.args.world_size
142
        if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
143
            start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
144
        output.metrics.update(
145
            speed_metrics(
146
                metric_key_prefix,
147
                start_time,
148
                num_samples=output.num_samples,
149
                num_steps=math.ceil(output.num_samples / total_batch_size),
150
            )
151
        )
152
        if self.post_process_function is None or self.compute_metrics is None:
153
            return output
154

155
        predictions = self.post_process_function(predict_examples, predict_dataset, output, "predict")
156
        metrics = self.compute_metrics(predictions)
157

158
        # Prefix all keys with metric_key_prefix + '_'
159
        for key in list(metrics.keys()):
160
            if not key.startswith(f"{metric_key_prefix}_"):
161
                metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
162
        metrics.update(output.metrics)
163
        return PredictionOutput(predictions=predictions.predictions, label_ids=predictions.label_ids, metrics=metrics)
164