class_practice.py - main - kroshko/omd

class_practice.py
130 строк4 KB
kroshko
08 ноя 2024, 20:11
08 ноя 2024, 20:112a34d7f
import math


class CountVectorizer:
    def __init__(self):
        self.feature_names = []

    def fit_transform(self, corpus):
        unique_words = set()
        for document in corpus:
            words = document.lower().split()
            unique_words.update(words)
        self.feature_names = sorted(list(unique_words))
        count_matrix = []
        for document in corpus:
            words = document.lower().split()
            word_count = [words.count(word) for word in self.feature_names]
            count_matrix.append(word_count)
        return count_matrix

    def get_feature_names(self):
        return self.feature_names


print('Задание 1')


if __name__ == '__main__':
    corpus = [
        'Crock Pot Pasta Never boil pasta again',
        'Pasta Pomodoro Fresh ingredients Parmesan to taste'
        ]
    vectorizer = CountVectorizer()
    count_matrix = vectorizer.fit_transform(corpus)
    print(vectorizer.get_feature_names())
    print(count_matrix)

# Задание 2
print('Задание 2')


def tf_transform(count_matrix):
    tf_matrix = []
    for count_v in count_matrix:
        total = sum(count_v)
        if total > 0:
            tf_vector = [round(count / total, 3) for count in count_v]
        else:
            tf_vector = [0] * len(count_v)
        tf_matrix.append(tf_vector)
    return tf_matrix


count_matrix = [
    [1, 1, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0],
    [0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1]
]
tf_matrix = tf_transform(count_matrix)

print(tf_matrix)

# Задание 3
print('Задание 3')


def idf_transform(count_matrix):
    return [round(math.log((len(count_matrix) + 1) / (sum(1 for v in count_matrix if v[i] > 0) + 1)) + 1, 3) for i in range(len(count_matrix[0]))]


idf_matrix = idf_transform(count_matrix)
print(idf_matrix)

# Задание 4
print('Задание 4')


class TfidfTransformer:
    def tf_transform(self, count_matrix):
        tf_matrix = []
        for count_v in count_matrix:
            total = sum(count_v)
            if total > 0:
                tf_vector = [round(count / total, 3) for count in count_v]
            else:
                tf_vector = [0] * len(count_v)
            tf_matrix.append(tf_vector)
        return tf_matrix

    def idf_transform(self, count_matrix):
        return [round(math.log((len(count_matrix) + 1) / (sum(1 for v in count_matrix if v[i] > 0) + 1)) + 1, 3) for i in range(len(count_matrix[0]))]

    def fit_transform(self, count_matrix):
        tf_matrix = self.tf_transform(count_matrix)
        idf_vector = self.idf_transform(count_matrix)
        return [[round(tf * idf, 3) for tf, idf in zip(tf_vector, idf_vector)] for tf_vector in tf_matrix]


count_matrix = [
    [1, 1, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0],
    [0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1]
]

transformer = TfidfTransformer()
tfidf_matrix = transformer.fit_transform(count_matrix)
print(tfidf_matrix)
print('Задание 5')


class TfidfVectorizer:
    def __init__(self):
        self.count_vectorizer = CountVectorizer()
        self.tfidf_transformer = TfidfTransformer()

    def fit_transform(self, corpus):
        count_matrix = self.count_vectorizer.fit_transform(corpus)
        return self.tfidf_transformer.fit_transform(count_matrix)

    def get_feature_names(self):
        return self.count_vectorizer.get_feature_names()


# Пример использования
corpus = [
    'Crock Pot Pasta Never boil pasta again',
    'Pasta Pomodoro Fresh ingredients Parmesan to taste'
]

vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(corpus)
print(tfidf_matrix)