financial-assistant

1
import nltk, pymorphy2, re
2
from nltk.corpus import stopwords
3

4
def remove_punctuation(text):
5
    text = re.sub(r'\*+', '', text)
6
    return text
7

8
def remove_stop_words(tokens):
9
    stop_words = set(stopwords.words('russian'))
10
    tagged_tokens = nltk.pos_tag(tokens, lang='rus')
11
    filtered_tokens = [word for word, tag in tagged_tokens if tag not in ['CONJ', 'PR', 'ADV'] and word.lower() not in stop_words]
12
    
13
    lemmas = [nltk.stem.WordNetLemmatizer().lemmatize(t) for t in filtered_tokens]
14
    return lemmas
15

16
def to_nominative_case(words):
17
    morph = pymorphy2.MorphAnalyzer()
18
    nominative_words = []
19
    for word in words:
20
        parsed_word = morph.parse(word)[0] 
21
        inflected_word = parsed_word.inflect({'nomn'})
22
        nominative_word = inflected_word.word if inflected_word else word 
23
        nominative_words.append(nominative_word)
24
    return nominative_words
25

26
def preprocessing(string):
27
  tokens = nltk.word_tokenize(string)
28
  lemmas = remove_stop_words(tokens)
29
  nom_lemmas = to_nominative_case(lemmas)
30
  preprocessed = ' '.join(nom_lemmas)
31
  cleaned_text = remove_punctuation(preprocessed)
32
  return cleaned_text