src/evaluation.py - master - Dinky6/Assessment-Question-Difficulty-Classifier

src/evaluation.py
88 строк3 KB
Dinky6
create src/evaluation.py
28 дек 2025, 17:14
28 дек 2025, 17:14f5e4383
from sklearn.metrics import (
    classification_report, confusion_matrix, 
    accuracy_score, precision_score, recall_score, f1_score,
    roc_auc_score
)
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import json

class ModelEvaluator:
    """Оценка и визуализация результатов модели"""
    
    def __init__(self, model, encoder):
        self.model = model
        self.encoder = encoder
        self.class_names = encoder.classes_ if encoder else ['easy', 'medium', 'hard']
    
    def evaluate(self, X_test, y_test, save_path=None):
        """Полная оценка модели"""
        y_pred = self.model.predict(X_test)
        y_proba = self.model.predict_proba(X_test) if hasattr(self.model, 'predict_proba') else None
        
        metrics = {
            'accuracy': accuracy_score(y_test, y_pred),
            'precision': precision_score(y_test, y_pred, average='weighted'),
            'recall': recall_score(y_test, y_pred, average='weighted'),
            'f1_score': f1_score(y_test, y_pred, average='weighted')
        }
        
        if y_proba is not None and len(np.unique(y_test)) > 2:
            try:
                metrics['roc_auc'] = roc_auc_score(
                    y_test, y_proba, multi_class='ovr', average='weighted'
                )
            except:
                metrics['roc_auc'] = None
        
        # Подробный отчет
        report = classification_report(
            y_test, y_pred, 
            target_names=self.class_names,
            output_dict=True
        )
        
        # Confusion matrix
        cm = confusion_matrix(y_test, y_pred)
        
        if save_path:
            self.save_metrics(metrics, save_path)
            self.plot_confusion_matrix(cm, save_path.replace('.json', '_cm.png'))
        
        return metrics, report, cm
    
    def save_metrics(self, metrics, path='models/metrics.json'):
        """Сохранение метрик в JSON"""
        with open(path, 'w') as f:
            json.dump(metrics, f, indent=4)
        print(f"Metrics saved to {path}")
    
    def plot_confusion_matrix(self, cm, path='models/confusion_matrix.png'):
        """Визуализация матрицы ошибок"""
        plt.figure(figsize=(8, 6))
        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
                    xticklabels=self.class_names,
                    yticklabels=self.class_names)
        plt.title('Confusion Matrix')
        plt.ylabel('True Label')
        plt.xlabel('Predicted Label')
        plt.tight_layout()
        plt.savefig(path, dpi=300)
        plt.close()
    
    def plot_feature_importance(self, model, feature_names, path='models/feature_importance.png'):
        """Визуализация важности признаков (для Random Forest)"""
        if hasattr(model, 'feature_importances_'):
            importances = model.feature_importances_
            indices = np.argsort(importances)[::-1]
            
            plt.figure(figsize=(10, 6))
            plt.title("Feature Importances")
            plt.bar(range(len(importances)), importances[indices])
            plt.xticks(range(len(importances)), 
                      [feature_names[i] for i in indices], rotation=45, ha='right')
            plt.tight_layout()
            plt.savefig(path, dpi=300)
            plt.close()