google-research

1
# coding=utf-8
2
# Copyright 2024 The Google Research Authors.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15

16
"""Tests for score_lib."""
17

18
from __future__ import absolute_import
19
from __future__ import division
20
from __future__ import print_function
21

22
from absl.testing import absltest
23
from wiki_split_bleu_eval import score_lib
24

25

26
class ScoreLibTest(absltest.TestCase):
27

28
  def setUp(self):
29
    super(ScoreLibTest, self).setUp()
30
    # Predictions.
31
    self.pred_data = [
32
        "aSrc ppnt jBL9 rSubRHTb . <::::> 9IU2 eH1V J6k17QPB jBL9 WIdmKeId.",
33
        "BLOI jBL9 zNWW Qn6p TJXXiRjg vpbG eH1V TTFk ."
34
    ]
35

36
    # Gold references.
37
    self.gold_data = [
38
        "\t".join([
39
            "aSrc ppnt jBL9 U6TlcN . <::::> 9IU2 eH1V J6k17QPB 3zIH lBCy .",
40
            "aSrc ppnt jBL9 U6TlcN poyY eH1V J6k17QPB WIdmKeId ."
41
        ]), "\t".join(["aSrc TTFk jBL9 77SJ poyY akLg jBL9 Qn6p TJXXiRjg ."])
42
    ]
43

44
  def test_non_parallel_data(self):
45
    """Validates detection of non-parallel predictions and gold items."""
46
    self.gold_data = self.gold_data[:1]
47
    gold = score_lib.ReadParcels(self.gold_data)
48
    pred = score_lib.ReadParcels(self.pred_data, reduce_to_single_analysis=True)
49
    with self.assertRaises(AssertionError) as cm:
50
      _ = score_lib.PerformEval(gold=gold, pred=pred)
51
      self.assertTrue(cm.exception.message.startswith("Got unequal"))
52

53
  def test_validate_scoring(self):
54
    """Simple test to validate scoring."""
55

56
    gold = score_lib.ReadParcels(self.gold_data)
57
    pred = score_lib.ReadParcels(self.pred_data, reduce_to_single_analysis=True)
58

59
    results = score_lib.PerformEval(gold=gold, pred=pred)
60
    print(results)
61

62
    # Compare against previously captured results.
63
    results = {k: round(v, 2) for k, v in results.items()}
64
    self.assertDictEqual(
65
        {
66
            "bleu.corpus.decomp": 24.20,
67
            "bleu.macro_avg_sent.decomp": 19.00,
68
            "counts.gold_inputs": 2,
69
            "counts.pred_inputs": 2,
70
            "counts.predictions": 2,
71
            "counts.references": 3,
72
            "lengths.simple_per_complex": 1.5,
73
            "lengths.tokens_per_simple": 7.0,
74
            "lengths.tokens_per_simple_micro": 6.33,
75
            "ref_lengths.simple_per_complex": 1.25,
76
            "ref_lengths.tokens_per_simple": 8.62,
77
            "refs_per_input.avg": 1.5,
78
            "refs_per_input.max": 2,
79
            "refs_per_input.min": 1,
80
            "uniq_refs_per_input.avg": 1.5,
81
            "uniq_refs_per_input.max": 2,
82
            "uniq_refs_per_input.min": 1
83
        }, results)
84

85

86
if __name__ == "__main__":
87
  absltest.main()
88