analyzer/create_graph_matrix.py - comb - obuxxxov/CodeForge

analyzer/create_graph_matrix.py
273 строки11 KB
ObliInSe
корректные пути в файле matrix_generated
19 авг 2025, 00:32
19 авг 2025, 00:32678a7aa
import pandas as pd
import json
import os
import re
from pathlib import Path

def clean_reference_markers(text):
    """
    Removes reference markers like [1], [2], [1-3], [1, 2] etc. from text
    """
    if pd.isna(text) or not text:
        return ""
    
    text = str(text)
    # Remove patterns like [1], [2], [1-3], [1, 2], [1–3] etc.
    # Pattern explanation:
    # \[ - literal opening bracket
    # (?:[\d\s,–-]+) - non-capturing group with digits, spaces, commas, dashes
    # \] - literal closing bracket
    cleaned_text = re.sub(r'\[(?:[\d\s,–-]+)\]', '', text)
    
    # Clean up extra spaces
    cleaned_text = re.sub(r'\s+', ' ', cleaned_text).strip()
    
    return cleaned_text

def convert_json_to_simple_text(topics_text):
    """
    Converts JSON format topics to simple comma-separated text
    """
    if pd.isna(topics_text) or not topics_text:
        return ""
    
    text = str(topics_text).strip()
    
    # If it's already simple text (no JSON), just clean and return
    if not (text.startswith('{') or text.startswith('[')):
        return clean_reference_markers(text)
    
    try:
        # Try to parse as JSON object
        if text.startswith('{'):
            topics_dict = json.loads(text)
            # Extract keys (topic names) and join with commas
            topic_names = list(topics_dict.keys())
            clean_topics = [clean_reference_markers(topic) for topic in topic_names]
            return ", ".join([t for t in clean_topics if t])
        
        # Try to parse as JSON array
        elif text.startswith('['):
            topics_list = json.loads(text)
            if isinstance(topics_list, list):
                clean_topics = [clean_reference_markers(str(topic)) for topic in topics_list]
                return ", ".join([t for t in clean_topics if t])
            
    except json.JSONDecodeError:
        # If JSON parsing fails, treat as simple text
        pass
    
    # Fallback: clean reference markers and return
    return clean_reference_markers(text)

def create_graph_matrix():
    """
    Creates graph_matrix.xlsx based on existing project data
    """
    
    print("🚀 Creating graph_matrix_generated.xlsx...")
    
    # Load data from competency_with_learning_topics_transformed.xlsx
    try:
        df_source = pd.read_excel('competency_with_learning_topics_transformed.xlsx')
        print(f"✅ Loaded {len(df_source)} competencies from competency_with_learning_topics_transformed.xlsx")
    except Exception as e:
        print(f"❌ Error loading source data: {e}")
        return
    
    # Create data structure for graph_matrix
    graph_data = []
    
    for idx, row in df_source.iterrows():
        # Basic data
        block = row.get('Блок', '')
        competency = row.get('Компетенция', '')
        ai_score = row.get('Оценка AI', 0)
        learning_topics = row.get('Темы для изучения', '')
        
        # Convert JSON to simple text and clean reference markers
        learning_topics_clean = convert_json_to_simple_text(learning_topics)
        
        # Create short version of learning topics
        short_topics = extract_short_topics(learning_topics_clean)
        
        # Find corresponding articles with ORDERED matching
        article_paths = find_article_paths_ordered(short_topics)
        
        # Create progress tracking array
        progress_array = create_progress_array(short_topics)
        
        # Form record
        record = {
            'Блок': block,
            'Компетенция': competency,
            'Оценка': '',              # Empty field for manual filling
            'Оценка Тимлида': '',      # Empty field for manual filling
            'Оценка AI': ai_score,     # Use AI score from transformed file
            'Темы для изучения кратко': short_topics,
            'Пути к статьям': article_paths,
            'Темы для изучения': learning_topics_clean,  # Use cleaned version
            'Рекомендация AI': '',     # Stub for AI recommendation
            'Текст Рекомендация AI': '',  # Empty - will be implemented later
            'Пройдено': progress_array # Use progress array
        }
        
        graph_data.append(record)
    
    # Create DataFrame and save to Excel (simple version)
    df_graph = pd.DataFrame(graph_data)
    df_graph.to_excel('graph_matrix_generated.xlsx', index=False)
    
    # Подсчитываем статистику по найденным статьям
    total_competencies = len(df_graph)
    competencies_with_articles = sum(1 for record in graph_data if record['Пути к статьям'].strip())
    competencies_without_articles = total_competencies - competencies_with_articles
    
    # Показываем примеры упорядоченного сопоставления
    print("\n📋 Примеры упорядоченного сопоставления тем и статей:")
    for i, record in enumerate(graph_data[:3]):  # Показываем первые 3 записи
        topics = record['Темы для изучения кратко']
        paths = record['Пути к статьям']
        print(f"  {i+1}. Темы: {topics}")
        print(f"     Пути: {paths}")
        print()
    
    print(f"✅ Created graph_matrix_generated.xlsx with {len(df_graph)} records")
    print("✅ Cleaned reference markers like [1], [2] from learning topics")
    print("✅ Converted JSON format to simple comma-separated text")
    print("✅ Added 'Пройдено' column with zeros array matching topic count")
    print(f"📊 Статистика сопоставления статей:")
    print(f"   - Всего компетенций: {total_competencies}")
    print(f"   - Найдены статьи: {competencies_with_articles}")
    print(f"   - Статьи не найдены: {competencies_without_articles}")
    
    return df_graph

def extract_short_topics(topics_text):
    """
    Extracts all topic names from full text (comma-separated format)
    """
    if pd.isna(topics_text) or not topics_text:
        return ""
    
    # Split and take ALL topics, not just first 3
    topics = [t.strip() for t in str(topics_text).split(',')]
    clean_topics = [clean_reference_markers(topic) for topic in topics]
    return ", ".join([t for t in clean_topics if t])  # All non-empty topics

def create_progress_array(short_topics):
    """
    Creates an array of zeros corresponding to the number of topics in short_topics
    """
    if not short_topics or short_topics.strip() == "":
        return ""  # Empty if no topics
    

    topic_count = short_topics.count(',') + 1
    
    # Create array of zeros as comma-separated string

    return ", ".join(["0"] * topic_count)

def find_article_paths(competency, short_topics):
    """
    Finds paths to created articles using EXACT matching via topic_info.json
    """
    articles_dir = Path("out")  # Change from "articles" to "out"
    if not articles_dir.exists():
        return ""
    
    found_paths = []
    
    # Get the absolute workspace path
    workspace_path = Path.cwd()  # Current working directory
    
    # Search for article directories that might correspond to competency or topics
    for article_dir in articles_dir.iterdir():
        if article_dir.is_dir():
            # Check if final article exists
            final_article = article_dir / "article_final.md"
            topic_info_file = article_dir / "topic_info.json"
            
            if final_article.exists() and topic_info_file.exists():
                try:
                    # Load topic info for exact matching
                    with open(topic_info_file, 'r', encoding='utf-8') as f:
                        topic_info = json.load(f)
                    
                    # Get exact topic name from JSON
                    exact_topic_name = topic_info.get('topic_name', '').strip()
                    
                    # Convert competency and topics for comparison
                    competency_normalized = competency.lower().strip()
                    topics_list = [topic.strip().lower() for topic in short_topics.split(',') if topic.strip()]
                    
                    # EXACT matching: check if competency or any topic exactly matches the article topic
                    if (competency_normalized == exact_topic_name.lower() or
                        any(topic.lower() == exact_topic_name.lower() for topic in topics_list)):
                        # Create full absolute path
                        full_path = workspace_path / final_article
                        found_paths.append(str(full_path))
                    # Убираем детальный вывод - только общая статистика
                        
                except Exception as e:
                    # Убираем детальный вывод ошибок - только общая статистика
                    continue
    
    return ", ".join(found_paths)  # Change separator from ";" to ","

def find_article_paths_ordered(short_topics):
    """
    Finds paths to created articles with ORDERED matching to maintain topic order
    """
    if not short_topics or short_topics.strip() == "":
        return ""
    
    # Split topics and clean them
    topics_list = [topic.strip() for topic in short_topics.split(',') if topic.strip()]
    if not topics_list:
        return ""
    
    articles_dir = Path("out")
    if not articles_dir.exists():
        return ""
    
    # Get the absolute workspace path
    workspace_path = Path.cwd()
    
    # Create ordered paths list - one path per topic in the same order
    ordered_paths = []
    
    for topic in topics_list:
        topic_path = ""  # Default: no article found for this topic
        
        # Search for article that matches this specific topic
        for article_dir in articles_dir.iterdir():
            if article_dir.is_dir():
                final_article = article_dir / "article_final.md"
                topic_info_file = article_dir / "topic_info.json"
                
                if final_article.exists() and topic_info_file.exists():
                    try:
                        with open(topic_info_file, 'r', encoding='utf-8') as f:
                            topic_info = json.load(f)
                        
                        exact_topic_name = topic_info.get('topic_name', '').strip()
                        
                        # EXACT match for this specific topic
                        if topic.lower().strip() == exact_topic_name.lower().strip():
                            full_path = str(workspace_path / final_article)
                            topic_path = full_path
                            break  # Found article for this topic, move to next
                            
                    except Exception as e:
                        continue
        
        # Add path (or empty string if no article found)
        ordered_paths.append(topic_path)
    
    # Join with commas - maintaining the exact order of topics
    return ", ".join(ordered_paths)

if __name__ == "__main__":
    create_graph_matrix()