/
ilmerkul
/
CountVectorizer
Обзор
Документация
Войти
/
ilmerkul
/
CountVectorizer
Код
Запросы
1
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
main.py
46 строк
2 KB
Ilya Merkulov
fix after review
27 окт 2024, 14:34
27 окт 2024, 14:34
7b4f85e
Код
Авторство
О чём код?
from typing import List, NoReturn import re class CountVectorizer: def __init__(self, lowercase=True): self.feature_names = None self.word2id = None self.lowercase = lowercase @staticmethod def tokenize(sens: str, lowercase: bool = True) -> List[str]: pattern = r"[A-Z]{2,}(?![a-z])|[A-Z][a-z]+(?=[A-Z])|[\'\w\-]+" result = re.findall(pattern, sens) if lowercase: result = list(map(str.lower, result)) return result def fit(self, corpus: List[str]) -> NoReturn: self.word2id = dict() self.feature_names = list() for sens in corpus: for word in self.tokenize(sens, self.lowercase): if word not in self.word2id: self.word2id[word] = len(self.feature_names) self.feature_names.append(word) def transform(self, corpus: List[str]) -> List[List[int]]: assert self.feature_names is not None and self.word2id is not None count_matrix = [[0] * len(self.feature_names) for _ in range(len(corpus))] for i, sens in enumerate(corpus): for word in self.tokenize(sens, self.lowercase): count_matrix[i][self.word2id[word]] += 1 return count_matrix def fit_transform(self, corpus: List[str]) -> List[List[int]]: self.fit(corpus) return self.transform(corpus) def get_feature_names(self) -> List[str]: assert self.feature_names is not None return self.feature_names