src/data_preprocessing.py - master - Dinky6/Assessment-Question-Difficulty-Classifier

src/data_preprocessing.py
54 строки2 KB
Dinky6
create src/data_preprocessing.py
28 дек 2025, 17:13
28 дек 2025, 17:1394bbe83
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, StandardScaler
import re

def load_data(filepath='data/questions.csv'):
    """Загрузка и предобработка данных"""
    df = pd.read_csv(filepath)
    
    # Очистка текста
    df['question_text'] = df['question_text'].apply(clean_text)
    
    # Кодирование категориальной переменной
    encoder = LabelEncoder()
    df['difficulty_encoded'] = encoder.fit_transform(df['difficulty_level'])
    
    return df, encoder

def clean_text(text):
    """Очистка текста вопроса"""
    if pd.isna(text):
        return ""
    text = str(text).lower()
    text = re.sub(r'[^\w\s?]', '', text)  # Удаляем пунктуацию кроме ?
    text = re.sub(r'\s+', ' ', text)  # Удаляем лишние пробелы
    return text.strip()

def prepare_features(df):
    """Подготовка признаков"""
    features = df[['avg_response_time_sec', 'correct_answer_rate']].copy()
    
    # Добавляем текстовые признаки
    features['text_length'] = df['question_text'].apply(len)
    features['word_count'] = df['question_text'].apply(lambda x: len(x.split()))
    features['question_mark'] = df['question_text'].apply(lambda x: 1 if '?' in x else 0)
    
    return features

def split_data(features, target, test_size=0.2, random_state=42):
    """Разделение данных на train/test"""
    X_train, X_test, y_train, y_test = train_test_split(
        features, target, 
        test_size=test_size, 
        random_state=random_state,
        stratify=target
    )
    
    # Масштабирование числовых признаков
    scaler = StandardScaler()
    X_train_scaled = scaler.fit_transform(X_train)
    X_test_scaled = scaler.transform(X_test)
    
    return X_train_scaled, X_test_scaled, y_train, y_test, scaler