src/feature_extraction.py - master - Dinky6/Assessment-Question-Difficulty-Classifier

src/feature_extraction.py
71 строка2 KB
Dinky6
create src/feature_extraction.py
28 дек 2025, 17:13
28 дек 2025, 17:13d2121fd
import textstat
import numpy as np
from typing import List, Dict

class TextFeatureExtractor:
    """Извлечение лингвистических признаков из текста"""
    
    def __init__(self):
        self.feature_names = [
            'flesch_reading_ease',
            'flesch_kincaid_grade',
            'gunning_fog',
            'coleman_liau',
            'dale_chall',
            'text_length',
            'word_count',
            'sentence_count',
            'avg_word_length',
            'avg_sentence_length',
            'question_mark'
        ]
    
    def extract_features(self, texts: List[str]) -> np.ndarray:
        """Извлечение всех признаков"""
        features = []
        
        for text in texts:
            text_features = self._extract_single_text_features(text)
            features.append(text_features)
        
        return np.array(features)
    
    def _extract_single_text_features(self, text: str) -> List[float]:
        """Извлечение признаков для одного текста"""
        features = []
        
        # Метрики читабельности
        features.append(textstat.flesch_reading_ease(text))
        features.append(textstat.flesch_kincaid_grade(text))
        features.append(textstat.gunning_fog(text))
        features.append(textstat.coleman_liau_index(text))
        features.append(textstat.dale_chall_readability_score(text))
        
        # Статистики текста
        features.append(len(text))
        features.append(len(text.split()))
        features.append(textstat.sentence_count(text))
        
        # Средняя длина слов
        words = text.split()
        if len(words) > 0:
            avg_word_len = sum(len(w) for w in words) / len(words)
        else:
            avg_word_len = 0
        features.append(avg_word_len)
        
        # Средняя длина предложений
        sentences = textstat.sentence_count(text)
        if sentences > 0:
            avg_sentence_len = len(words) / sentences
        else:
            avg_sentence_len = 0
        features.append(avg_sentence_len)
        
        # Наличие вопросительного знака
        features.append(1 if '?' in text else 0)
        
        return features
    
    def get_feature_names(self) -> List[str]:
        return self.feature_names