data/sample_data_generator.py - master - katherinesiv/study_material_recommender

data/sample_data_generator.py
454 строки18 KB
Сиваева Екатерина
update data/sample_data_generator.py
24 дек 2025, 20:02
24 дек 2025, 20:02ee8f479
"""
Data generator for creating sample study materials and ratings
"""

import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import random
from typing import List, Dict, Any
import json

# Seed for reproducibility
random.seed(42)
np.random.seed(42)


class SampleDataGenerator:
    """Generates sample data for study materials recommender system"""
    
    def __init__(self):
        self.materials_data = []
        self.ratings_data = []
        
        # Define categories and properties
        self.subjects = [
            "Computer Science", "Mathematics", "Statistics", 
            "Physics", "Design", "Ethics", "Game Development",
            "Finance", "Biology", "Data Science"
        ]
        
        self.difficulties = ["beginner", "intermediate", "advanced"]
        
        self.material_types = ["book", "video", "course", "article"]
        
        self.keywords_pool = {
            "Computer Science": [
                "python", "programming", "algorithms", "machine learning", "ai",
                "web development", "database", "software engineering", "cloud",
                "security", "data structures", "computer vision", "nlp"
            ],
            "Mathematics": [
                "linear algebra", "calculus", "probability", "statistics",
                "math fundamentals", "equations", "theorem", "proof"
            ],
            "Data Science": [
                "data analysis", "visualization", "big data", "analytics",
                "data mining", "predictive modeling"
            ]
        }
        
    def generate_materials(self, n_materials: int = 100) -> pd.DataFrame:
        """
        Generate sample study materials.
        
        Args:
            n_materials: Number of materials to generate
            
        Returns:
            DataFrame with materials data
        """
        print(f"Generating {n_materials} sample materials...")
        
        for i in range(1, n_materials + 1):
            subject = random.choice(self.subjects)
            difficulty = random.choice(self.difficulties)
            material_type = random.choice(self.material_types)
            
            # Generate appropriate keywords based on subject
            if subject in self.keywords_pool:
                keywords = ", ".join(random.sample(self.keywords_pool[subject], 
                                                  random.randint(2, 4)))
            else:
                keywords = "general, learning, education"
            
            # Generate material properties based on type
            if material_type == "book":
                duration = random.randint(60, 300)
                pages = random.randint(100, 500)
            elif material_type == "video":
                duration = random.randint(10, 120)
                pages = 0
            elif material_type == "course":
                duration = random.randint(60, 240)
                pages = random.randint(50, 200)
            else:  # article
                duration = random.randint(5, 60)
                pages = random.randint(10, 50)
            
            # Determine level based on difficulty
            if difficulty == "beginner":
                level = random.randint(1, 2)
            elif difficulty == "intermediate":
                level = random.randint(2, 3)
            else:  # advanced
                level = random.randint(3, 5)
            
            material = {
                "material_id": i,
                "title": self._generate_title(subject, difficulty, material_type),
                "description": self._generate_description(subject, difficulty),
                "subject": subject,
                "difficulty": difficulty,
                "material_type": material_type,
                "duration_minutes": duration,
                "pages": pages,
                "level": level,
                "keywords": keywords
            }
            
            self.materials_data.append(material)
        
        materials_df = pd.DataFrame(self.materials_data)
        print(f"Generated {len(materials_df)} materials")
        return materials_df
    
    def _generate_title(self, subject: str, difficulty: str, 
                       material_type: str) -> str:
        """Generate a title for a material"""
        prefixes = {
            "Computer Science": ["Introduction to", "Advanced", "Fundamentals of",
                                "Complete Guide to", "Essentials of"],
            "Mathematics": ["Basic", "Applied", "Theoretical", "Practical"],
            "beginner": ["Getting Started with", "Learn", "Basics of"],
            "intermediate": ["Deep Dive into", "Mastering", "Understanding"],
            "advanced": ["Advanced Topics in", "Expert", "Cutting-edge"]
        }
        
        topics = {
            "Computer Science": ["Python Programming", "Machine Learning", 
                                "Web Development", "Algorithms", "Databases",
                                "Computer Vision", "Natural Language Processing"],
            "Mathematics": ["Linear Algebra", "Calculus", "Probability Theory",
                          "Statistics", "Discrete Mathematics"]
        }
        
        if subject in topics:
            topic = random.choice(topics[subject])
        else:
            topic = subject
            
        prefix_options = []
        if subject in prefixes:
            prefix_options.extend(prefixes[subject])
        if difficulty in prefixes:
            prefix_options.extend(prefixes[difficulty])
            
        if prefix_options:
            prefix = random.choice(prefix_options)
            title = f"{prefix} {topic}"
        else:
            title = f"{topic} {difficulty.capitalize()} Course"
            
        # Add material type hint
        if material_type == "book":
            title += " (Book)"
        elif material_type == "video":
            title += " (Video Tutorial)"
        elif material_type == "course":
            title += " (Online Course)"
            
        return title
    
    def _generate_description(self, subject: str, difficulty: str) -> str:
        """Generate a description for a material"""
        templates = [
            f"This {difficulty} level course covers essential concepts in {subject}. "
            f"Perfect for students looking to build strong foundations.",
            
            f"A comprehensive {difficulty} guide to {subject}. "
            f"Learn practical skills and theoretical knowledge.",
            
            f"Master {subject} with this {difficulty} level material. "
            f"Includes examples, exercises, and real-world applications.",
            
            f"This {difficulty} resource provides in-depth understanding of {subject}. "
            f"Suitable for both academic and professional development."
        ]
        
        return random.choice(templates)
    
    def generate_ratings(self, n_users: int = 20, 
                        n_materials: int = 100,
                        rating_density: float = 0.2) -> pd.DataFrame:
        """
        Generate sample ratings data.
        
        Args:
            n_users: Number of users to generate ratings for
            n_materials: Number of materials available
            rating_density: Proportion of possible user-material ratings to generate
            
        Returns:
            DataFrame with ratings data
        """
        print(f"Generating ratings for {n_users} users...")
        
        # Generate base timestamp
        base_date = datetime(2024, 1, 1)
        
        # Define user interests (simulate different user profiles)
        user_interests = {}
        for user_id in range(1, n_users + 1):
            # Each user has preference for certain subjects and difficulties
            preferred_subjects = random.sample(self.subjects, 
                                             random.randint(1, 3))
            preferred_difficulty = random.choice(self.difficulties)
            user_interests[user_id] = {
                "subjects": preferred_subjects,
                "difficulty": preferred_difficulty
            }
        
        ratings_count = 0
        for user_id in range(1, n_users + 1):
            # Each user rates a subset of materials
            n_ratings_for_user = int(n_materials * rating_density)
            materials_to_rate = random.sample(range(1, n_materials + 1), 
                                            n_ratings_for_user)
            
            for material_id in materials_to_rate:
                # Generate rating based on user interests
                rating = self._generate_rating(user_id, material_id, 
                                             user_interests[user_id])
                
                # Generate timestamp (increasing over time)
                days_offset = random.randint(0, 120)
                hours_offset = random.randint(0, 23)
                minutes_offset = random.randint(0, 59)
                
                timestamp = (base_date + 
                           timedelta(days=days_offset,
                                    hours=hours_offset,
                                    minutes=minutes_offset))
                
                rating_record = {
                    "user_id": user_id,
                    "material_id": material_id,
                    "rating": rating,
                    "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S")
                }
                
                self.ratings_data.append(rating_record)
                ratings_count += 1
        
        ratings_df = pd.DataFrame(self.ratings_data)
        print(f"Generated {len(ratings_df)} ratings")
        
        # Calculate and print statistics
        actual_density = len(ratings_df) / (n_users * n_materials)
        print(f"Actual rating density: {actual_density:.3f}")
        print(f"Average rating: {ratings_df['rating'].mean():.2f}")
        
        return ratings_df
    
    def _generate_rating(self, user_id: int, material_id: int, 
                        user_interests: Dict[str, Any]) -> float:
        """Generate a rating based on user interests and material properties"""
        # Find material info
        material = None
        for m in self.materials_data:
            if m["material_id"] == material_id:
                material = m
                break
        
        if not material:
            # Default rating if material not found
            return random.randint(3, 5)
        
        # Base rating influenced by user interests
        base_rating = 3.0
        
        # Adjust based on subject match
        if material["subject"] in user_interests["subjects"]:
            base_rating += 1.0
        
        # Adjust based on difficulty match
        if material["difficulty"] == user_interests["difficulty"]:
            base_rating += 0.5
        elif (user_interests["difficulty"] == "beginner" and 
              material["difficulty"] == "advanced"):
            base_rating -= 1.0
        elif (user_interests["difficulty"] == "advanced" and 
              material["difficulty"] == "beginner"):
            base_rating -= 0.5
        
        # Add some randomness
        rating = base_rating + random.uniform(-0.5, 0.5)
        
        # Clip to 1-5 range and round to nearest 0.5
        rating = max(1.0, min(5.0, rating))
        rating = round(rating * 2) / 2
        
        return rating
    
    def generate_interaction_data(self, n_interactions: int = 1000) -> pd.DataFrame:
        """
        Generate additional interaction data (clicks, views, completions).
        
        Args:
            n_interactions: Number of interactions to generate
            
        Returns:
            DataFrame with interaction data
        """
        print(f"Generating {n_interactions} interactions...")
        
        interactions = []
        interaction_types = ["view", "click", "start", "complete", "bookmark"]
        
        for _ in range(n_interactions):
            user_id = random.randint(1, 20)
            material_id = random.randint(1, 100)
            interaction_type = random.choice(interaction_types)
            
            # Generate timestamp within last 30 days
            days_ago = random.randint(0, 30)
            hours_ago = random.randint(0, 23)
            minutes_ago = random.randint(0, 59)
            
            timestamp = (datetime.now() - 
                        timedelta(days=days_ago,
                                 hours=hours_ago,
                                 minutes=minutes_ago))
            
            interaction = {
                "user_id": user_id,
                "material_id": material_id,
                "interaction_type": interaction_type,
                "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S"),
                "duration_seconds": random.randint(10, 600) if interaction_type in ["view", "start"] else 0
            }
            
            interactions.append(interaction)
        
        interactions_df = pd.DataFrame(interactions)
        print(f"Generated {len(interactions_df)} interactions")
        
        return interactions_df
    
    def save_to_csv(self, materials_path: str = "materials.csv",
                   ratings_path: str = "ratings.csv",
                   interactions_path: str = None):
        """
        Save generated data to CSV files.
        
        Args:
            materials_path: Path to save materials CSV
            ratings_path: Path to save ratings CSV
            interactions_path: Path to save interactions CSV (optional)
        """
        # Save materials
        if self.materials_data:
            materials_df = pd.DataFrame(self.materials_data)
            materials_df.to_csv(materials_path, index=False)
            print(f"Saved materials to {materials_path}")
        
        # Save ratings
        if self.ratings_data:
            ratings_df = pd.DataFrame(self.ratings_data)
            ratings_df.to_csv(ratings_path, index=False)
            print(f"Saved ratings to {ratings_path}")
        
        # Save interactions if generated
        if interactions_path:
            interactions_df = self.generate_interaction_data()
            interactions_df.to_csv(interactions_path, index=False)
            print(f"Saved interactions to {interactions_path}")
    
    def generate_sample_dataset(self, output_dir: str = "."):
        """
        Generate a complete sample dataset.
        
        Args:
            output_dir: Directory to save generated files
        """
        print("=" * 50)
        print("Generating sample dataset for Study Material Recommender")
        print("=" * 50)
        
        # Generate data
        materials_df = self.generate_materials(40)
        ratings_df = self.generate_ratings(20, 40, 0.2)
        
        # Save to CSV
        materials_path = f"{output_dir}/materials.csv"
        ratings_path = f"{output_dir}/ratings.csv"
        
        materials_df.to_csv(materials_path, index=False)
        ratings_df.to_csv(ratings_path, index=False)
        
        print("\nDataset Summary:")
        print(f"- Materials: {len(materials_df)}")
        print(f"- Ratings: {len(ratings_df)}")
        print(f"- Users: {ratings_df['user_id'].nunique()}")
        print(f"- Rating sparsity: {1 - (len(ratings_df) / (ratings_df['user_id'].nunique() * materials_df['material_id'].nunique())):.3f}")
        print(f"- Average rating: {ratings_df['rating'].mean():.2f}")
        print(f"\nFiles saved to:")
        print(f"  {materials_path}")
        print(f"  {ratings_path}")
        
        # Generate statistics file
        stats = {
            "dataset_info": {
                "name": "Study Materials Sample Dataset",
                "generation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                "version": "1.0"
            },
            "materials": {
                "count": len(materials_df),
                "subjects": materials_df["subject"].value_counts().to_dict(),
                "difficulties": materials_df["difficulty"].value_counts().to_dict(),
                "types": materials_df["material_type"].value_counts().to_dict()
            },
            "ratings": {
                "count": len(ratings_df),
                "unique_users": int(ratings_df["user_id"].nunique()),
                "average_rating": float(ratings_df["rating"].mean()),
                "rating_distribution": ratings_df["rating"].value_counts().sort_index().to_dict()
            }
        }
        
        stats_path = f"{output_dir}/dataset_statistics.json"
        with open(stats_path, 'w') as f:
            json.dump(stats, f, indent=2)
        
        print(f"  {stats_path}")
        print("\n" + "=" * 50)
        print("Dataset generation complete!")
        print("=" * 50)


def main():
    """Main function to generate sample data"""
    import argparse
    
    parser = argparse.ArgumentParser(description="Generate sample data for Study Material Recommender")
    parser.add_argument("--output-dir", default="data", help="Output directory for generated files")
    parser.add_argument("--n-materials", type=int, default=40, help="Number of materials to generate")
    parser.add_argument("--n-users", type=int, default=20, help="Number of users to generate")
    parser.add_argument("--rating-density", type=float, default=0.2, 
                       help="Density of ratings (proportion of possible user-material pairs)")
    
    args = parser.parse_args()
    
    # Create output directory if it doesn't exist
    import os
    os.makedirs(args.output_dir, exist_ok=True)
    
    # Generate data
    generator = SampleDataGenerator()
    generator.generate_sample_dataset(args.output_dir)


if __name__ == "__main__":
    main()