dream

1
# flake8: noqa
2
import requests
3
import difflib
4

5
SEED = 31415
6

7
true_requests = {}
8

9
false_requests = []
10
# Response struct
11
# [
12
#     {
13
#         "text": "t1.",
14
#         "hypotheses": [
15
#             {
16
#                 "skill_name": "game_cooperative_skill",
17
#                 "text": "b1.",
18
#                 "confidence": 1.0,
19
#                 "can_continue": "can",
20
#                 "state": {"1": 2},
21
#             }
22
#         ],
23
#     },
24
#     {"text": "b1.", "active_skill": "game_cooperative_skill", "confidence": 1.0},
25
#     {"text": "t2.", "hypotheses": []},
26
# ]
27

28

29
def update_utterances(utterances=None, response=None, text_request=""):
30
    utterances = [] if utterances is None else utterances
31
    if response:
32
        text, confidence, _, _, attr = response
33
        can_continue = attr.get("can_continue", "")
34
        utterances[-1]["hypotheses"] = [
35
            {
36
                "skill_name": "game_cooperative_skill",
37
                "text": text,
38
                "confidence": confidence,
39
                "can_continue": can_continue,
40
            }
41
        ]
42
        utterances += [
43
            {"text": text, "orig_text": text, "active_skill": "game_cooperative_skill", "confidence": confidence},
44
        ]
45
    if text_request:
46
        utterances += [
47
            {"text": text_request, "hypotheses": []},
48
        ]
49
    return utterances
50

51

52
request_utters = [
53
    "hi",
54
    "yeah sure",
55
    "top of last year",
56
    "go on",
57
    "move on",
58
    "top of last year",
59
    "yeah sure",
60
    "talk",
61
    "yeah sure",
62
    "ten",
63
    "next",
64
]
65
true_response_utters = [
66
    "So, well, one of my hobbies is keeping fresh stats about the top video games. Wanna dig into it?",
67
    "I can tell you a few things about popular games. For now, I can talk about the most popular games for this or last year, last month, or even the last week (hotties!). Which of these time periods is of interest for you?",
68
    "There were 20 newly released games highly rated in the last year. Do you want to learn more? ",
69
    "The game with the highest rating is Cyberpunk 2077. It was released on December 10 2020. It's a combination of  Action & RPG. 58 percent of people marked Cyberpunk 2077 as exceptional.  If you want to discuss it in details say I want to talk about it.  Chatting about it or the next one? ",
70
    "The next game is Fall Guys: Ultimate Knockout. It was released on August 04 2020. It's a combination of Action, Sports, Casual Massively Multiplayer & Indie. 60 percent of people marked Fall Guys: Ultimate Knockout as recommended.  Discussing it or moving on? ",
71
    "There were 20 newly released games highly rated in the last year. Do you want to learn more? ",
72
    "The game with the highest rating is Cyberpunk 2077. It was released on December 10 2020. It's a combination of  Action & RPG. 58 percent of people marked Cyberpunk 2077 as exceptional.  If you want to discuss it in details say I want to talk about it.  Discussing it or moving on? ",
73
    "Have you played it before? ",
74
    "So I suppose you liked Cyberpunk 2077 right? How would you rate the desire to play it again, from 1 to 10?",
75
    "You gave it a really high rating. Your rating is higher than one given by the rest of the players. Most of them rated it at 8.4 points. Well. I'd love to talk about other things but my developer forgot to add them to my memory banks. Please forgive him, he's young and very clever. For now can we please discuss the next game?",
76
    "The next game is Fall Guys: Ultimate Knockout. It was released on August 04 2020. It's a combination of Action, Sports, Casual Massively Multiplayer & Indie. 60 percent of people marked Fall Guys: Ultimate Knockout as recommended.  Discussing it or moving on? ",
77
]
78

79

80
def test_skill():
81
    url = "http://0.0.0.0:8068/respond"
82
    utterances = []
83
    warnings = 0
84
    human_attr = {}
85
    bot_attr = {}
86

87
    for ind, (req_utter, true_resp_utter) in enumerate(zip(request_utters, true_response_utters)):
88
        utterances = update_utterances(utterances=utterances, text_request=req_utter)
89
        human_utterances = [uttr for uttr in utterances if "hypotheses" in uttr]
90
        bot_utterances = [uttr for uttr in utterances if "hypotheses" not in uttr]
91
        input_data = {
92
            "dialogs": [
93
                {
94
                    "utterances": utterances,
95
                    "human_utterances": human_utterances,
96
                    "bot_utterances": bot_utterances,
97
                    "human": {"attributes": human_attr},
98
                    "bot": {"attributes": bot_attr},
99
                }
100
            ]
101
        }
102
        input_data["rand_seed"] = SEED + ind
103
        response = requests.post(url, json=input_data).json()[0]
104
        utterances = update_utterances(utterances=utterances, response=response)
105
        text, confidence, human_attr, bot_attr, attr = response
106
        ratio = difflib.SequenceMatcher(None, true_resp_utter.split(), text.split()).ratio()
107

108
        print("----------------------------------------")
109
        print(f"req_utter = {req_utter}")
110
        print(f"true_resp_utter = {true_resp_utter}")
111
        print(f"cand_resp_utter = {text}")
112
        print(f"ratio = {ratio}")
113

114
        if ratio < 0.35:
115
            warnings += 1
116
            print(f"warning={warnings}")
117
        # print(difflib.SequenceMatcher(None, true_resp_utter.split(), text.split()).ratio())
118
    assert warnings == 0
119
    print("SUCCESS!")
120

121

122
if __name__ == "__main__":
123
    test_skill()
124