scanStrings.py - master - Flipper/Marginal

scanStrings.py
613 строк19 KB
Alex
Ceased tracking app_metadata.json and buildManifest.json. Now replaced with template files.
29 янв 2026, 15:03
29 янв 2026, 15:03cb3a72b
"""
Scanner for translatable strings that integrates with utils module.
Exports to locales folder with timestamped directory.

[NOTICE] This script is fully AI-generated. To be replaces/edited in later versions
"""

import sys
import os
import ast
import re
from pathlib import Path
from datetime import datetime

# Add src to path to import utils
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))

from src.utils import getBasePath, listDir


class DualStringExtractor(ast.NodeVisitor):
    """
    AST visitor that extracts translatable strings from Python code.
    Handles both UI strings (_(), self.translate()) and log messages (log(), customLog()).
    """

    def __init__(self, basePath: str):
        """
        Initialize the string extractor.

        Args:
            basePath (str): Base directory path for relative file paths.
        """
        self.basePath = basePath
        self.uiStrings = []  # For _() and self.translate() calls
        self.logStrings = []  # For log() and customLog() calls
        self.currentFile = ""

    def visit_Call(self, node):
        """
        Visit function call nodes to extract translatable strings.

        Args:
            node (ast.Call): AST call node to examine.

        Returns:
            None
        """
        # Check for translation function calls: _("string")
        if isinstance(node.func, ast.Name):
            funcName = node.func.id
            if funcName == "_":
                self._extractUiString(node, "global _()")
            elif funcName == "log":
                self._extractLogString(node, "log()")
            elif funcName == "customLog":
                self._extractCustomLogString(node, "customLog()")

        # Check for method calls: self.translate(), obj.log(), obj.customLog(), etc.
        elif isinstance(node.func, ast.Attribute):
            attrName = node.func.attr

            if attrName == "log":
                # Get context string for the object (self, logger, etc.)
                context = self._getCallContext(node)
                self._extractLogString(node, context)

            elif attrName == "customLog":
                # Get context string for the object (self.logger, etc.)
                context = self._getCallContext(node)
                self._extractCustomLogString(node, context)

            elif attrName in ["translate", "_"]:
                # Check if it's self.translate or obj.translate
                context = self._getCallContext(node)
                self._extractUiString(node, context)

        self.generic_visit(node)

    def _getCallContext(self, node) -> str:
        """
        Generate a readable context string for method calls.

        Args:
            node (ast.Call): AST call node.

        Returns:
            str -> String representation of the call chain (e.g., "self.translate()").
        """
        try:
            # Build string representation of the call chain
            parts = []
            current = node.func

            while isinstance(current, ast.Attribute):
                parts.append(current.attr)
                current = current.value

            if isinstance(current, ast.Name):
                parts.append(current.id)
            elif isinstance(current, ast.Call):
                # Handle cases like obj().method()
                parts.append("object()")
            else:
                parts.append("object")

            parts.reverse()  # Put object first, then methods
            return ".".join(parts) + "()"

        except Exception:
            return "method_call()"

    def _extractUiString(self, node, context: str = ""):
        """
        Extract strings from _() and self.translate() calls.

        Args:
            node (ast.Call): AST call node containing the string.
            context (str): Context description for the call.

        Returns:
            None
        """
        if node.args and isinstance(node.args[0], ast.Constant):
            if isinstance(node.args[0].value, str):
                string = node.args[0].value
                isFormat = self._hasFormatPlaceholders(string)

                self.uiStrings.append({
                    'file': self.currentFile,
                    'line': node.lineno,
                    'string': string,
                    'type': 'ui',
                    'context': context,
                    'isFormat': isFormat
                })

    def _extractLogString(self, node, context: str = ""):
        """
        Extract strings from log() calls.

        Args:
            node (ast.Call): AST call node containing the string.
            context (str): Context description for the call.

        Returns:
            None
        """
        author = None
        message = None

        # Try to find message argument
        for i, arg in enumerate(node.args):
            if i == 0 and isinstance(arg, ast.Constant) and isinstance(arg.value, str):
                author = arg.value
            elif i == 1 and isinstance(arg, ast.Constant) and isinstance(arg.value, str):
                message = arg.value
                break

        # Also check keyword arguments
        for kw in node.keywords:
            if kw.arg == "message" and isinstance(kw.value, ast.Constant):
                if isinstance(kw.value.value, str):
                    message = kw.value.value
            elif kw.arg == "author" and isinstance(kw.value, ast.Constant):
                if isinstance(kw.value.value, str):
                    author = kw.value.value

        if message:
            isFormat = self._hasFormatPlaceholders(message)

            self.logStrings.append({
                'file': self.currentFile,
                'line': node.lineno,
                'author': author or "Unknown",
                'string': message,
                'type': 'log',
                'context': context,
                'isFormat': isFormat
            })

    def _extractCustomLogString(self, node, context: str = ""):
        """
        Extract strings from customLog() calls.

        Args:
            node (ast.Call): AST call node containing the string.
            context (str): Context description for the call.

        Returns:
            None
        """
        content = None
        translate = True  # Default value is True

        # Extract all string constants from arguments
        stringArgs = []
        for arg in node.args:
            if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
                stringArgs.append(arg.value)

        # Check keyword arguments
        for kw in node.keywords:
            if kw.arg == "content" and isinstance(kw.value, ast.Constant):
                if isinstance(kw.value.value, str):
                    content = kw.value.value
            elif kw.arg == "translate" and isinstance(kw.value, ast.Constant):
                if isinstance(kw.value.value, bool):
                    translate = kw.value.value
            elif kw.arg in ["message", "text", "msg"] and isinstance(kw.value, ast.Constant):
                if isinstance(kw.value.value, str) and not content:
                    content = kw.value.value

        # If no content found in keywords, check positional arguments
        if not content and stringArgs:
            # Skip self argument for method calls
            if "." in context and stringArgs and stringArgs[0] != "self":
                content = stringArgs[0]
            elif not context.startswith("self.") and stringArgs:
                content = stringArgs[0]

        if content and translate:
            isFormat = self._hasFormatPlaceholders(content)

            self.logStrings.append({
                'file': self.currentFile,
                'line': node.lineno,
                'author': "customLog",
                'string': content,
                'type': 'log',
                'context': context,
                'isFormat': isFormat
            })

    def _hasFormatPlaceholders(self, string: str) -> bool:
        """
        Check if string has {} or {named} placeholders.

        Args:
            string (str): String to check for format placeholders.

        Returns:
            bool -> True if string contains format placeholders.
        """
        placeholderPattern = r'\{[^{}]*\}'
        return bool(re.search(placeholderPattern, string))

    def scanFile(self, filepath: str):
        """
        Scan a single Python file for translatable strings.

        Args:
            filepath (str): Path to Python file to scan.

        Returns:
            None
        """
        self.currentFile = str(Path(filepath).relative_to(self.basePath))
        try:
            with open(filepath, 'r', encoding='utf-8') as f:
                content = f.read()

            tree = ast.parse(content, filename=filepath)
            self.visit(tree)

        except (SyntaxError, UnicodeDecodeError) as e:
            print(f"[WARNING] Could not parse {filepath}: {e}")


def scanProject(sourceDir: str = None):
    """
    Scan entire project directory using utils.listDir.

    Args:
        sourceDir (str, optional): Source directory to scan (defaults to base path).

    Returns:
        DualStringExtractor -> Instance with extracted strings.
    """
    if sourceDir is None:
        sourceDir = getBasePath()

    extractor = DualStringExtractor(sourceDir)

    # Use utils.listDir to get all Python files recursively
    pyFiles = listDir(
        folderPath=sourceDir,
        pattern="*.py",
        recursive=True,
        fullPath=True
    )

    # Filter out excluded directories
    excludeDirs = ['__pycache__', '.venv', 'venv', 'locales', 'builds']
    filteredFiles = []

    for filePath in pyFiles:
        # Skip files in excluded directories
        skip = False
        for exclude in excludeDirs:
            if f"/{exclude}/" in filePath.replace("\\", "/") or f"\\{exclude}\\" in filePath:
                skip = True
                break
        if not skip:
            filteredFiles.append(filePath)

    print(f"[INFO] Found {len(filteredFiles)} Python files to scan")

    for pyFile in filteredFiles:
        print(f"[SCAN] Scanning: {Path(pyFile).relative_to(sourceDir)}")
        extractor.scanFile(pyFile)

    return extractor


def getManualStrings():
    """
    Return the manually added strings that should always be included.

    Returns:
        list[dict] -> List of manual string dictionaries.
    """
    return [
        {
            'file': 'src\\UI\\SettingsDialog.py',
            'line': "NA",
            'string': "Never",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\SettingsDialog.py',
            'line': "NA",
            'string': "Once per action",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\SettingsDialog.py',
            'line': "NA",
            'string': "Once per session",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\SettingsDialog.py',
            'line': "NA",
            'string': "Comma ','",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\SettingsDialog.py',
            'line': "NA",
            'string': "Semicolon ';'",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\SettingsDialog.py',
            'line': "NA",
            'string': "Pipe '|'",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\SettingsDialog.py',
            'line': "NA",
            'string': "Tabulation '\\t'",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\CustomUI\\DragNDropArea.py',
            'line': "NA",
            'string': "Open",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\CustomUI\\DragNDropArea.py',
            'line': "NA",
            'string': "Clear",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\CustomUI\\DragNDropArea.py',
            'line': "NA",
            'string': "Invalid file type",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\CustomUI\\LineDirSelector.py',
            'line': "NA",
            'string': "Open folder",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        },
        {
            'file': 'src\\UI\\CustomUI\\LineDirSelector.py',
            'line': "NA",
            'string': "Drag and drop a folder here or type path manually",
            'type': 'ui',
            'context': 'self.translate()',
            'isFormat': False
        }
    ]


def generateBuilderTemplate(mode='limited', existingPoFile=None, outputDir=None):
    """
    Simplified function for builder usage - generates messages.po and README.txt.

    Args:
        mode (str): Context mode ("limited" for builder).
        existingPoFile (str, optional): Path to existing .po file for merging.
        outputDir (str, optional): Output directory for .po file.

    Returns:
        bool -> True if successful, False otherwise.
    """
    print(f"[START] Starting translation string extraction for builder...")

    # Scan project for strings
    extractor = scanProject()

    if len(extractor.uiStrings) == 0 and len(extractor.logStrings) == 0:
        print("[ERROR] No translatable strings found!")
        return False

    # Create output directory
    os.makedirs(outputDir, exist_ok=True)

    # Get manual strings
    manualStrings = getManualStrings()

    # Generate messages.po file
    poFile = os.path.join(outputDir, "messages.po")
    _writePoFile(poFile, extractor.uiStrings + extractor.logStrings, "All Messages", mode, manualStrings)

    # Create a README file with instructions
    readmePath = os.path.join(outputDir, "README.txt")
    _createReadme(readmePath, extractor, mode)

    print(f"[SUCCESS] Generated messages.po - {len(extractor.uiStrings) + len(extractor.logStrings)} total strings")
    print(f"[SUCCESS] Generated README.txt - Instructions file")

    return True


def _writePoFile(filename, strings, title, mode="full", manualStrings=None):
    """
    Write strings to a .po file according to selected mode.

    Args:
        filename (str): Output .po file path.
        strings (list[dict]): List of string dictionaries to write.
        title (str): Title for the .po file header.
        mode (str): Context mode ("no_context", "limited", or "full").
        manualStrings (list[dict], optional): List of manually added strings.

    Returns:
        None
    """
    if manualStrings is None:
        manualStrings = []

    poContent = [
        f'# {title}',
        '# This file is distributed under the same license as the "MARGINAL" application.',
        '# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.',
        '#',
        '#, fuzzy',
        'msgid ""',
        'msgstr ""',
        '"Project-Id-Version: PACKAGE VERSION\\n"',
        '"Report-Msgid-Bugs-To: \\n"',
        f'"POT-Creation-Date: {datetime.now().strftime("%Y-%m-%d %H:%M+0000")}\\n"',
        '"Language: \\n"',
        '"Content-Type: text/plain; charset=UTF-8\\n"',
        '"Content-Transfer-Encoding: 8bit\\n"',
        '',
    ]

    # Add manual strings first (always with full context)
    for item in manualStrings:
        string = item.get('string', '')
        if not string or string.isspace():
            continue

        # Escape quotes and newlines
        escaped = string.replace('"', '\\"').replace('\n', '\\n')

        poContent.extend([
            f'#. {item.get("context", "self.translate()")}',
            f'#: {item["file"]}:{item["line"]}',
            f'msgid "{escaped}"',
            'msgstr ""',
            '',
        ])

    # Add extracted strings according to mode
    for item in strings:
        string = item.get('string', '')
        if not string or string.isspace():
            continue

        # Escape quotes and newlines
        escaped = string.replace('"', '\\"').replace('\n', '\\n')

        # Add context according to mode
        if mode == "no_context":
            # No context at all, just the string
            pass
        elif mode == "limited":
            # Only type and context, no file/line
            if item.get('type') == 'log':
                author = item.get('author', 'Unknown')
                poContent.append(f"#. Log message from {author}")
                if item.get('isFormat'):
                    poContent.append("#. Note: Contains format placeholders")
            else:
                context = item.get('context', 'UI string')
                poContent.append(f"#. {context}")
                if item.get('isFormat'):
                    poContent.append("#. Note: Contains format placeholders")
        else:  # full context
            if item.get('type') == 'log':
                author = item.get('author', 'Unknown')
                context = item.get('context', '')
                contextStr = f" ({context})" if context else ""
                poContent.append(f"#. Log message from {author}{contextStr}")
                if item.get('isFormat'):
                    poContent.append("#. Note: Contains format placeholders")
                poContent.append(f'#: {item["file"]}:{item["line"]}')
            else:
                context = item.get('context', 'UI string')
                poContent.append(f"#. {context}")
                if item.get('isFormat'):
                    poContent.append("#. Note: Contains format placeholders")
                poContent.append(f'#: {item["file"]}:{item["line"]}')

        poContent.append(f'msgid "{escaped}"')
        poContent.append('msgstr ""')
        poContent.append('')

    with open(filename, 'w', encoding='utf-8') as f:
        f.write('\n'.join(poContent))


def _createReadme(readmePath, extractor, mode):
    """
    Create a README file with instructions.

    Args:
        readmePath (str): Path to README file.
        extractor (DualStringExtractor): Extractor with scanned strings.
        mode (str): Context mode used.

    Returns:
        None
    """
    with open(readmePath, 'w', encoding='utf-8') as f:
        f.write(f"""Translation Template Export
============================
Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}

How to use:
1. Create a language folder with the language's code in '...\\resources\\locales\\'
2. Copy this .po file to your language folder
3. Translate the msgstr fields
4. Rename .po file to marginal.po
5. Compile with: msgfmt messages.po -o messages.mo
6. Add new language to manifest.json file under the availableLocales

Note: Format strings (with {{}}) must preserve placeholders!
""")


def main():
    """
    Main entry point for command-line usage.

    Returns:
        None
    """
    try:
        # For builder usage, just generate the template
        print("This script is intended to be called from BUILD.py")
        print("Run BUILD.py to generate locale templates.")

    except Exception as e:
        print(f"\n[ERROR] Error: {e}")
        import traceback
        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()