dream

1
# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
import pathlib
16
import uuid
17
import logging
18

19
from programy.clients.embed.basic import EmbeddedDataFileBot
20

21
from common.dialogflow_framework.programy.text_preprocessing import clean_text
22

23
logger = logging.getLogger(__name__)
24

25

26
class MindfulDataFileBot(EmbeddedDataFileBot):
27
    def ask_question(self, userid, question):
28
        client_context = self.create_client_context(userid)
29
        return self.renderer.render(client_context, self.process_question(client_context, question))
30

31
    def __call__(self, human_utterances, clean_response=False):
32
        userid = uuid.uuid4().hex
33
        for uttr in human_utterances:
34
            uttr = clean_text(uttr)
35
            response = self.ask_question(userid, uttr)
36
        response = response if response else ""
37
        response = clean_text(response) if clean_response else response
38
        return response
39

40

41
def get_configuration_files(
42
    aiml_dirs=None,
43
    learnf_dirs=None,
44
    sets_dirs=None,
45
    maps_dirs=None,
46
    rdfs_dirs=None,
47
    conversations_dir=None,
48
    services_dir=None,
49
    properties_txt_file=None,
50
    defaults_txt_file=None,
51
    denormals_txt_file=None,
52
    normals_txt_file=None,
53
    genders_txt_file=None,
54
    persons_txt_file=None,
55
    person2s_txt_file=None,
56
    triggers_txt_file=None,
57
    regexes_txt_file=None,
58
    spellings_txt_file=None,
59
    duplicates_txt_file=None,
60
    errors_txt_file=None,
61
    licenses_keys_file=None,
62
    usergroups_yaml_file=None,
63
    preprocessors_conf_file=None,
64
    postprocessors_conf_file=None,
65
    postquestionprocessors_conf_file=None,
66
):
67
    # example:
68
    # ############# From storage
69
    # aiml_dirs -> aiml = [".../storage/categories"],
70
    # learnf_dirs -> learnf = [".../storage/categories/learnf"],
71
    # sets_dirs -> sets = [".../storage/sets"],
72
    # maps_dirs -> maps = [".../storage/maps"],
73
    # rdfs_dirs -> rdfs = [".../storage/rdfs"],
74
    # conversations_dir -> conversations = ".../storage/conversations",
75
    # services_dir -> services = ".../storage/services",
76
    # ############# From properties
77
    # properties_txt_file -> properties = ".../storage/properties/properties.txt",
78
    # defaults_txt_file -> defaults = ".../storage/properties/defaults.txt",
79
    # ############# From lookups
80
    # denormals_txt_file -> denormals = ".../storage/lookups/denormal.txt",
81
    # normals_txt_file -> normals = ".../storage/lookups/normal.txt",
82
    # genders_txt_file -> genders = ".../storage/lookups/gender.txt",
83
    # persons_txt_file -> persons = ".../storage/lookups/person.txt",
84
    # person2s_txt_file -> person2s = ".../storage/lookups/person2.txt",
85
    # ############# From triggers
86
    # triggers_txt_file -> triggers = ".../storage/triggers/triggers.txt",
87
    # ############# From regex
88
    # regexes_txt_file -> regexes = ".../storage/regex/regex-templates.txt",
89
    # ############# From spelling
90
    # spellings_txt_file -> spellings = ".../storage/spelling/corpus.txt",
91
    # ############# From debug
92
    # duplicates_txt_file -> duplicates = ".../storage/debug/duplicates.txt",
93
    # errors_txt_file -> errors = ".../storage/debug/errors.txt",
94
    # ############# From licenses
95
    # licenses_keys_file -> licenses = ".../storage/licenses/license.keys",
96
    # ############# From security
97
    # usergroups_yaml_file -> usergroups = ".../storage/security/usergroups.yaml",
98
    # ############# From processing
99
    # preprocessors_conf_file -> preprocessors = ".../storage/processing/preprocessors.conf",
100
    # postprocessors_conf_file -> postprocessors = ".../storage/processing/postprocessors.conf",
101
    # postquestionprocessors_conf_file -> postquestionprocessors = ".../storage/processing/postquestionprocessors.conf",
102
    files = {}
103
    if aiml_dirs:
104
        files["aiml"] = aiml_dirs
105
    if learnf_dirs:
106
        files["learnf"] = learnf_dirs
107
    if sets_dirs:
108
        files["sets"] = sets_dirs
109
    if maps_dirs:
110
        files["maps"] = maps_dirs
111
    if rdfs_dirs:
112
        files["rdfs"] = rdfs_dirs
113
    if conversations_dir:
114
        files["conversations"] = conversations_dir
115
    if services_dir:
116
        files["services"] = services_dir
117
    if properties_txt_file:
118
        files["properties"] = properties_txt_file
119
    if defaults_txt_file:
120
        files["defaults"] = defaults_txt_file
121
    if denormals_txt_file:
122
        files["denormals"] = denormals_txt_file
123
    if normals_txt_file:
124
        files["normals"] = normals_txt_file
125
    if genders_txt_file:
126
        files["genders"] = genders_txt_file
127
    if persons_txt_file:
128
        files["persons"] = persons_txt_file
129
    if person2s_txt_file:
130
        files["person2s"] = person2s_txt_file
131
    if triggers_txt_file:
132
        files["triggers"] = triggers_txt_file
133
    if regexes_txt_file:
134
        files["regexes"] = regexes_txt_file
135
    if spellings_txt_file:
136
        files["spellings"] = spellings_txt_file
137
    if duplicates_txt_file:
138
        files["duplicates"] = duplicates_txt_file
139
    if errors_txt_file:
140
        files["errors"] = errors_txt_file
141
    if licenses_keys_file:
142
        files["licenses"] = licenses_keys_file
143
    if usergroups_yaml_file:
144
        files["usergroups"] = usergroups_yaml_file
145
    if preprocessors_conf_file:
146
        files["preprocessors"] = preprocessors_conf_file
147
    if postprocessors_conf_file:
148
        files["postprocessors"] = postprocessors_conf_file
149
    if postquestionprocessors_conf_file:
150
        files["postquestionprocessors"] = postquestionprocessors_conf_file
151

152
    dropped_files = {}
153
    for name, file in files.items():
154
        if isinstance(file, list):
155
            sub_files = [pathlib.Path(sub_file) for sub_file in file]
156
            sub_files = [sub_file for sub_file in sub_files if not sub_file.exists()]
157
            if sub_files:
158
                dropped_files[name] = sub_files
159
        elif not pathlib.Path(file).exists():
160
            dropped_files[name] = pathlib.Path(file)
161
    if dropped_files:
162
        raise Exception(f"dropped_files={dropped_files} is not empty")
163
    return files
164

165

166
def get_programy_model(configuration_files):
167
    model = MindfulDataFileBot(configuration_files)
168
    return model
169