CSS-LM
85 строк · 2.9 Кб
1# coding=utf-8
2# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
3# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17try:
18from scipy.stats import pearsonr, spearmanr
19from sklearn.metrics import matthews_corrcoef, f1_score
20
21_has_sklearn = True
22except (AttributeError, ImportError):
23_has_sklearn = False
24
25
26def is_sklearn_available():
27return _has_sklearn
28
29
30if _has_sklearn:
31
32def simple_accuracy(preds, labels):
33return (preds == labels).mean()
34
35def acc_and_f1(preds, labels):
36acc = simple_accuracy(preds, labels)
37f1 = f1_score(y_true=labels, y_pred=preds)
38return {
39"acc": acc,
40"f1": f1,
41"acc_and_f1": (acc + f1) / 2,
42}
43
44def pearson_and_spearman(preds, labels):
45pearson_corr = pearsonr(preds, labels)[0]
46spearman_corr = spearmanr(preds, labels)[0]
47return {
48"pearson": pearson_corr,
49"spearmanr": spearman_corr,
50"corr": (pearson_corr + spearman_corr) / 2,
51}
52
53def glue_compute_metrics(task_name, preds, labels):
54assert len(preds) == len(labels)
55if task_name == "cola":
56return {"mcc": matthews_corrcoef(labels, preds)}
57elif task_name == "sst-2":
58return {"acc": simple_accuracy(preds, labels)}
59elif task_name == "mrpc":
60return acc_and_f1(preds, labels)
61elif task_name == "sts-b":
62return pearson_and_spearman(preds, labels)
63elif task_name == "qqp":
64return acc_and_f1(preds, labels)
65elif task_name == "mnli":
66return {"mnli/acc": simple_accuracy(preds, labels)}
67elif task_name == "mnli-mm":
68return {"mnli-mm/acc": simple_accuracy(preds, labels)}
69elif task_name == "qnli":
70return {"acc": simple_accuracy(preds, labels)}
71elif task_name == "rte":
72return {"acc": simple_accuracy(preds, labels)}
73elif task_name == "wnli":
74return {"acc": simple_accuracy(preds, labels)}
75elif task_name == "hans":
76return {"acc": simple_accuracy(preds, labels)}
77else:
78raise KeyError(task_name)
79
80def xnli_compute_metrics(task_name, preds, labels):
81assert len(preds) == len(labels)
82if task_name == "xnli":
83return {"acc": simple_accuracy(preds, labels)}
84else:
85raise KeyError(task_name)
86