classes.py - practise_classes - idaria/omd

classes.py
102 строки3 KB
daria
Сделала практику по классам
14 ноя 2024, 03:52
14 ноя 2024, 03:52f4ab8cb
import math

class CountVectorizer:
    def __init__(self):
        self.uniq_words = list()

    def fit_transform(self, texts):
        uniq_words = list()
        for text in texts:
            for word in text.split():
                word = word.lower()
                if word not in uniq_words:
                    uniq_words.append(word)

        count_count_matrix = list()
        for text in texts:
            word_counter = dict()
            for word in text.split():
                word = word.lower()
                if word not in word_counter:
                    word_counter[word] = 1
                else:
                    word_counter[word] += 1

            count_list = list()
            for word in uniq_words:
                word = word.lower()
                if word in word_counter:
                    count_list.append(word_counter[word])
                else:
                    count_list.append(0)
            count_count_matrix.append(count_list)
        self.uniq_words = uniq_words
        return count_count_matrix

    def get_feature_names(self):
        return self.uniq_words


class TfidfTransformer:
    def tf_transform(self, matrix):
        total_num = list()
        for row in matrix:
            total_num.append(sum(row))

        result = list()
        for i in range(len(matrix)):
            row_result = []
            for val in matrix[i]:
                row_result.append(round(val / total_num[i], 3))
            result.append(row_result)
        return result

    def idf_transform(self, matrix):
        docs_count = len(matrix)
        terms_count = [0] * len(matrix[0])

        for row in matrix:
            for i, val in enumerate(row):
                if val > 0:
                    terms_count[i] += 1

        result = []
        for count in terms_count:
            idf = round(math.log((docs_count + 1) / (count + 1)) + 1, 1)
            result.append(idf)

        return result

    def fit_transform(self, matrix):
        tf_matrix = self.tf_transform(matrix)
        idf_vector = self.idf_transform(matrix)

        result = []
        for tf_row in tf_matrix:
            new_row = []
            for i in range(len(tf_row)):
                new_row.append(round(tf_row[i] * idf_vector[i], 3))
            result.append(new_row)

        return result


class TfidfVectorizer(CountVectorizer):
    def __init__(self):
        super().__init__()
        self.transformer = TfidfTransformer()

    def fit_transform(self, texts):
        count_matrix = super().fit_transform(texts)
        return self.transformer.fit_transform(count_matrix)


if __name__ == '__main__':
    vectorizer = TfidfVectorizer()
    corpus = [
        'Crock Pot Pasta Never boil pasta again',
        'Pasta Pomodoro Fresh ingredients Parmesan to taste'
    ]

    print(vectorizer.fit_transform(corpus))
    print(vectorizer.get_feature_names())