mteb

1
from __future__ import annotations
2

3
from mteb.evaluation.evaluators import InstructionRetrievalEvaluator, utils
4

5

6
class TestInstructionRetrievalEvaluator:
7
    def setup_method(self):
8
        """Setup any state tied to the execution of the given method in a class.
9

10
        setup_method is invoked for every test method of a class.
11
        """
12
        # checks that it loads
13
        self.evaluator = InstructionRetrievalEvaluator.InstructionRetrievalEvaluator()
14

15
    def test_p_mrr(self):
16
        changed_qrels = {
17
            "a": ["0"],
18
        }
19

20
        # these are the query: {"doc_id": score}
21
        original_run = {
22
            "a": {"0": 1, "1": 2, "2": 3, "3": 4},
23
        }
24

25
        new_run = {
26
            "a": {"0": 1, "1": 2, "2": 3, "3": 4},
27
        }
28

29
        results = utils.evaluate_change(
30
            original_run,
31
            new_run,
32
            changed_qrels,
33
        )
34

35
        assert results["p-MRR"] == 0.0
36

37
        # test with a change
38

39
        new_run = {
40
            "a": {"0": 4, "1": 1, "2": 2, "3": 3},
41
        }
42

43
        results = utils.evaluate_change(
44
            original_run,
45
            new_run,
46
            changed_qrels,
47
        )
48

49
        assert results["p-MRR"] == -0.75
50

51
        # test with a positive change
52

53
        results = utils.evaluate_change(
54
            new_run,
55
            original_run,
56
            changed_qrels,
57
        )
58

59
        assert results["p-MRR"] == 0.75
60