src/modules/repoExporter.py - master - Doggich/CodeRepresentation

src/modules/repoExporter.py
260 строк11 KB
Doggich
refactoring: script structure
10 янв 2026, 19:23
10 янв 2026, 19:236336d86
import requests
import base64
from datetime import datetime

class GitHubRepoExporter:
    """
    A comprehensive exporter for GitHub repositories that retrieves and saves
    the complete file structure and contents of a repository.
    
    This class handles API communication with GitHub, recursively traverses
    repository directories, extracts file contents, and generates a formatted
    text document containing the entire repository structure with file contents.
    
    ## Attributes:
        owner (str): Repository owner/organization
        repo (str): Repository name
        base_url (str): GitHub API base URL
        headers (dict): HTTP headers for API requests
        total_files (int): Counter for total files processed
        total_size (int): Total size of all files in bytes
        skipped_files (int): Count of files that couldn't be processed
    """
    
    def __init__(self, owner, repo, token=None):
        """
        Initialize the GitHub repository exporter.
        
        Args:
            owner (str): Repository owner username or organization name
            repo (str): Repository name
            token (str, optional): GitHub personal access token for increased
                                 API rate limits and access to private repos
        """
        self.owner = owner
        self.repo = repo
        self.base_url = "https://api.github.com"
        self.headers = {
            'Accept': 'application/vnd.github.v3+json',
            'User-Agent': 'GitHubRepoExporter/1.0'
        }
        
        if token:
            self.headers['Authorization'] = f'token {token}'
        
        self.total_files = 0
        self.total_size = 0
        self.skipped_files = 0
        
    def _make_request(self, url):
        """
        Execute HTTP GET request to GitHub API with error handling.
        
        Args:
            url (str): Full URL for the API request
            
        Returns:
            requests.Response or None: Response object if successful,
                                     None if request failed
        """
        try:
            response = requests.get(url, headers=self.headers)
            
            # Monitor API rate limits
            if 'X-RateLimit-Remaining' in response.headers:
                remaining = int(response.headers['X-RateLimit-Remaining'])
                if remaining < 10:
                    print(f"Warning: API requests remaining: {remaining}")
            
            return response
        except Exception as e:
            print(f"Request error: {e}")
            return None
    
    def get_file_content(self, path):
        """
        Retrieve and decode the content of a specific file from the repository.
        
        Args:
            path (str): Relative path to the file within the repository
            
        Returns:
            str or None: Decoded file content as string, or None if file
                        is a directory, binary, or cannot be accessed
        """
        url = f"{self.base_url}/repos/{self.owner}/{self.repo}/contents/{path}"
        response = self._make_request(url)
        
        if not response or response.status_code != 200:
            print(f"Failed to retrieve file {path}: {response.status_code if response else 'No response'}")
            return None
        
        file_info = response.json()
        
        # If response is a list, it's a directory
        if isinstance(file_info, list):
            return None
        
        # Try to decode base64 content
        if 'content' in file_info and file_info['content']:
            try:
                content = base64.b64decode(file_info['content']).decode('utf-8')
                return content
            except:
                # Fall back to download_url for large or problematic files
                pass
        
        # Alternative method using direct download URL
        if 'download_url' in file_info and file_info['download_url']:
            try:
                content_response = requests.get(file_info['download_url'])
                if content_response.status_code == 200:
                    # Attempt to decode as text
                    try:
                        return content_response.text
                    except:
                        return f"[BINARY FILE - size: {file_info.get('size', 0)} bytes]"
            except:
                pass
        
        return None
    
    def is_text_file(self, filename):
        """
        Determine if a file is likely to be a text file based on its extension.
        
        Args:
            filename (str): Name of the file to check
            
        Returns:
            bool: True if file extension matches known text file extensions,
                  False otherwise
        """
        text_extensions = [
            # Programming languages
            '.py', '.js', '.java', '.cpp', '.c', '.cs', '.go', '.rs', '.php', '.rb', 
            '.swift', '.kt', '.scala', '.hs', '.lua', '.pl', '.sh', '.bat', '.ps1',
            # Web technologies
            '.html', '.htm', '.css', '.scss', '.sass', '.less', '.jsx', '.tsx', '.ts',
            # Data formats
            '.json', '.xml', '.yaml', '.yml', '.toml', '.ini', '.cfg', '.conf',
            # Documentation
            '.md', '.txt', '.rst', '.tex', '.adoc',
            # Configuration files
            '.gitignore', '.dockerignore', '.editorconfig',
            # Other text formats
            '.csv', '.sql', '.log'
        ]
        
        return any(filename.lower().endswith(ext) for ext in text_extensions)
    
    def explore_repository(self, path="", output_file="repository_content.txt"):
        """
        Recursively traverse the entire repository structure and write all
        files and their contents to an output text file.
        
        This method performs a depth-first traversal of the repository,
        writing the directory structure and file contents in a human-readable
        format with line numbers for each file.
        
        Args:
            path (str): Starting path within repository (empty for root)
            output_file (str): Name of the output file to write results to
        """
        start_time = datetime.now()
        
        # Open output file for writing
        with open(output_file, 'w', encoding='utf-8') as f:
            # Report header
            f.write("=" * 80 + "\n")
            f.write(f"COMPLETE REPOSITORY OVERVIEW: {self.owner}/{self.repo}\n")
            f.write(f"Export date: {start_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
            f.write("=" * 80 + "\n\n")
            
            # Recursive function for directory traversal
            def process_directory(current_path="", indent=0):
                nonlocal f
                
                url = f"{self.base_url}/repos/{self.owner}/{self.repo}/contents/{current_path}"
                response = self._make_request(url)
                
                if not response or response.status_code != 200:
                    error_msg = f"[ACCESS ERROR FOR {current_path}]" if current_path else "[ACCESS ERROR FOR ROOT]"
                    f.write(f"{'  ' * indent}ERROR: {error_msg}\n")
                    return
                
                items = response.json()
                
                # Sort: directories first, then files
                for item in sorted(items, key=lambda x: (x['type'] != 'dir', x['name'].lower())):
                    if item['type'] == 'dir':
                        # Write directory information
                        dir_indent = '|   ' * indent + '|-- '
                        f.write(f"{dir_indent}DIR: {item['name']}/\n")
                        
                        # Recursively process subdirectory
                        process_directory(item['path'], indent + 1)
                        
                    else:
                        # This is a file
                        self.total_files += 1
                        file_size = item.get('size', 0)
                        self.total_size += file_size
                        
                        file_indent = '|   ' * indent + '|-- '
                        size_str = f" ({file_size} bytes)" if file_size > 0 else ""
                        
                        # Check if we can read this file type
                        if self.is_text_file(item['name']):
                            f.write(f"{file_indent}FILE: {item['name']}{size_str}\n")
                            
                            # Get file content
                            content = self.get_file_content(item['path'])
                            
                            if content:
                                # Write separator and content
                                content_indent = '|   ' * (indent + 1)
                                f.write(f"{content_indent}|-- {'-' * 40}\n")
                                f.write(f"{content_indent}|    FILE CONTENTS:\n")
                                f.write(f"{content_indent}|    {'-' * 40}\n")
                                
                                # Add line numbering
                                lines = content.split('\n')
                                for i, line in enumerate(lines, 1):
                                    line_num = f"{i:4d} | "
                                    f.write(f"{content_indent}|    {line_num}{line}\n")
                                
                                f.write(f"{content_indent}|    {'-' * 40}\n")
                                f.write(f"{content_indent}|\n")
                            else:
                                f.write(f"{content_indent}|-- [UNABLE TO READ FILE CONTENTS]\n\n")
                                self.skipped_files += 1
                        else:
                            # Skip binary or unsupported files
                            f.write(f"{file_indent}BIN: {item['name']}{size_str} [BINARY/UNSUPPORTED FILE TYPE]\n")
                            self.skipped_files += 1
            
            # Start traversal from root
            print(f"Starting repository traversal for {self.owner}/{self.repo}...")
            process_directory()
            
            # Write final statistics
            end_time = datetime.now()
            elapsed_time = (end_time - start_time).total_seconds()
            
            f.write("\n" + "=" * 80 + "\n")
            f.write("FINAL STATISTICS:\n")
            f.write("=" * 80 + "\n")
            f.write(f"* Total files in repository: {self.total_files}\n")
            f.write(f"* Files successfully read: {self.total_files - self.skipped_files}\n")
            f.write(f"* Files skipped (binary/errors): {self.skipped_files}\n")
            f.write(f"* Total size: {self.total_size:,} bytes\n")
            f.write(f"* Execution time: {elapsed_time:.2f} seconds\n")
            f.write(f"* Completion date: {end_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
            f.write("=" * 80 + "\n")
        
        print(f"Export completed! Results saved to {output_file}")
        print(f"  Total files: {self.total_files}")
        print(f"  Successfully read: {self.total_files - self.skipped_files}")
        print(f"  Skipped: {self.skipped_files}")
        print(f"  Time elapsed: {elapsed_time:.2f} seconds")