buildroot/share/scripts/languageExport.py - bugfix-2.1.x - githubmirror/Marlin

buildroot/share/scripts/languageExport.py
287 строк13 KB
Scott Lahteine
🌐 Language export -l argument
08 фев 2026, 23:39
08 фев 2026, 23:39bdc57af
#!/usr/bin/env python3
'''
languageExport.py [--single] [--translate]

Export LCD language strings to CSV files for easier translation.
Use languageImport.py to import CSV into the language files.

Use --single to export all languages to a single CSV file.
'''

import re, argparse
from pathlib import Path
from sys import argv, exit
from languageUtil import *

LANGHOME = "Marlin/src/lcd/language"
OUTDIR = Path('out-csv')

def language_export(args={}):
    # A dictionary to contain strings for each language.
    # Init with 'en' so English will always be first.
    language_strings = { 'en': {} }

    # A dictionary to contain all distinct LCD string names
    names = {}

    # Get all "language_*.h" files
    langfiles = sorted(list(Path(LANGHOME).glob('language_*.h')))

    # Read each language file
    for langfile in langfiles:
        # Get the language code from the filename
        langcode = langfile.name.replace('language_', '').replace('.h', '')

        # Skip 'test' and any others that we don't want
        if langcode in ['test']: continue

        # Allow space-delimited list or multiple arguments
        if args.language:
            language_args = args.language[0].split(' ') if ' ' in args.language[0] else args.language

        # Always load canonical US English and specified (or all other) languages
        if langcode != 'en' and language_args and langcode not in language_args: continue

        # Open the file
        f = open(langfile, 'r', encoding='utf-8')
        if not f: continue

        # Flags to indicate a wide or tall section
        wideflag, tallflag = False, False
        # A counter for the number of strings in the file
        stringcount = 0
        # A dictionary to hold all the strings
        strings = { 'narrow': {}, 'wide': {}, 'tall': {} }
        # Read each line in the file
        for line in f:
            # Clean up the line for easier parsing
            line = line.split("//")[0].strip()
            if line.endswith(';'): line = line[:-1].strip()

            # Check for wide or tall sections, assume no complicated nesting
            if line.startswith("#endif") or line.startswith("#else"):
                wideflag, tallflag = False, False
            elif re.match(r'#if.*WIDTH\s*>=?\s*2[01].*', line): wideflag = True
            elif re.match(r'#if.*LCD_HEIGHT\s*>=?\s*4.*', line): tallflag = True

            # For string-defining lines capture the string data
            match = re.match(r'LSTR\s+([A-Z0-9_]+)\s*=\s*(.+)\s*', line)
            if match:
                # Name and quote-sanitized value
                name, value = match.group(1), match.group(2).replace('\\"', '$$$')

                # Remove all _UxGT wrappers from the value in a non-greedy way
                value = re.sub(r'_UxGT\((".*?")\)', r'\1', value)

                # Multi-line strings get one or more bars | for identification
                multiline = 0
                multimatch = re.match(r'.*MSG_(\d)_LINE\s*\(\s*(.+?)\s*\).*', value)
                if multimatch:
                    multiline = int(multimatch.group(1))
                    value = '|' + re.sub(r'"\s*,\s*"', '|', multimatch.group(2))

                # Wrap inline defines in parentheses
                value = re.sub(r' *([A-Z0-9]+_[A-Z0-9_]+) *', r'(\1)', value)
                # Remove quotes around strings
                value = re.sub(r'"(.*?)"', r'\1', value).replace('$$$', '""')
                # Store all unique names as dictionary keys
                names[name] = 1
                # Store the string as narrow, wide, tall
                section = 'tall' if tallflag else 'wide' if wideflag else 'narrow'
                strings[section][name] = value

                # Increment the string counter
                stringcount += 1
                # Break for testing
                if args.limit and stringcount >= int(args.limit): break

        # Close the file
        f.close()
        # Store the array in the dict
        language_strings[langcode] = strings

    # Get the codes of all imported languages
    langcodes = list(language_strings.keys())

    if args.verbose:
        print("Languages:", ' '.join(langcodes))

    # Print the array
    #print(language_strings)

    # Report the total number of unique strings
    print("Found %s distinct LCD strings." % len(names))

    #exit(0)

    # Add missing translations, if specified
    if args.translate:

        MIN_TRANSLATE_LEN = 2
        NEVER_TRANSLATE_LANGS = ( 'el_CY', 'fr_na' )
        NEVER_TRANSLATE_NAMES = (
          "MSG_MARLIN", "MSG_CUSTOM_MENU_MAIN_TITLE",
          "MSG_PID_P", "MSG_PID_P_E",
          "MSG_PID_I", "MSG_PID_I_E",
          "MSG_PID_D", "MSG_PID_D_E",
          "MSG_PID_C", "MSG_PID_C_E",
          "MSG_PID_F", "MSG_PID_F_E",
          "MSG_BACKLASH_N",
          "MSG_SHORT_DAY", "MSG_SHORT_HOUR", "MSG_SHORT_MINUTE",
          "MSG_FTM_ZV", "MSG_FTM_ZVD", "MSG_FTM_ZVDD", "MSG_FTM_ZVDDD",
          "MSG_FTM_EI", "MSG_FTM_2HEI", "MSG_FTM_3HEI", "MSG_FTM_MZV"
        )

        import ollama

        DEFAULT_MODEL = (
          "qwen3:32b",                 #  0  22 GB
          "gpt-oss:20b",               #  1  13 GB
          "llama3.3",                  #  2  45 GB
          "deepseek-r1:14b",           #  3   9 GB
          "deepseek-r1-qwen-14b",      #  4  15 GB
          "devstral:24b",              #  5  15 GB
          "qwen3-coder:30b",           #  6  18 GB
          "mistral-small-3.2",         #  7  14 GB
          "openthinker:32b",           #  8  19 GB
          "deepseek-v2",               #  9   9 GB
          "deepseek-coder-v2",         # 10   9 GB
          "llama3.2:3b-instruct-fp16"  # 11   6 GB
        )[0]

        llm_model = args.model if args.model else DEFAULT_MODEL

        def get_system_prompt(args, sect):
            if sect == 'narrow':
                length_limit = "no more than 18 characters long! Use common abbreviations whenever necessary"
            elif sect == 'tall':
                length_limit = "no more than 3 strings of 20 characters. Use common abbreviations if necessary"
            else: # wide
                length_limit = "around the same length as the given example(s)"

            no_thinking = "" if args.dothink else "Do not think! Just translate.\n"

            system_prompt_text = f"""You are an expert in language translation in the context of 3D printing.
You will be given a list of existing translations and will be asked to provide a new translation in the given language.
When provided, the English (en) translation should be considered the most authoritative source.
Named variable substitutions are written as UPPERCASE_WITH_UNDERSCORES. Never translate or modify these!
The symbols `@`, `~`, `*`, `{{`, and `$` are special characters used for substitution. Never translate or modify these!
Your translations must be {length_limit}.
Assume that variable substitutions such as (MACHINE_NAME) are short strings for the purpose of character counting.
{no_thinking}For each translation requested, respond only with the translated string, no introduction, explanation, or assessment.
This clean output will be perfect for our use case."""

            return [{ 'role': 'system', 'content': system_prompt_text }]

        # Send a prompt to Ollama and return the reply text
        def prompt_with_ollama(SYSTEM_PROMPT, prompt:str):
            msg = [{ 'role': 'user', 'content': prompt }]
            response = ollama.chat(model=llm_model, messages=SYSTEM_PROMPT + msg, stream=False)
            reply = response['message']['content'].strip('\n')
            reply = re.sub(r'<think>[\s\S]+</think>\n*', '', reply)
            reply = re.sub(r'(^"|"$)', '', reply)
            return reply

        # For each named string fill in any missing translations
        for sect in ('narrow','wide','tall'):
            system_prompt = get_system_prompt(args, sect)
            for name in names.keys():
                if name in NEVER_TRANSLATE_NAMES: continue
                en_string = language_strings['en'][sect][name] if name in language_strings['en'][sect] else ""
                glyphs = len(en_string)

                done = {} # All existing translations for the given name
                todo = [] # Missing translation keys to create
                for lang in langcodes:
                    strings = language_strings[lang]
                    if name in strings[sect]:
                        done[lang] = strings[sect][name]
                    elif glyphs >= MIN_TRANSLATE_LEN and lang not in NEVER_TRANSLATE_LANGS:
                        todo += [lang]

                # For each untranslated language, fill in a translation
                for lang in todo:
                    # Show existing translations to the LLM and ask for one more
                    prompt = [ f"Please translate the following string into {language_name(lang)} ({lang})." ]
                    if lang.endswith("_na"):
                        prompt += [ "(Substitute plain unaccented ASCII characters for accented characters in the output.)" ]
                    prompt += [ "Here are the existing translations:" ]
                    for dlang in done.keys():
                        prompt += [ f"- {dlang} {language_name(dlang)}: \"{done[dlang]}\"" ]
                    prompt = '\n'.join(prompt)
                    #print(f"Prompt: {prompt}")
                    reply = prompt_with_ollama(system_prompt, prompt)
                    newstring = reply.replace('–','-').replace('‑','-').replace('／','/').replace('’',"'").replace('…','...').replace('\u202F',' ').replace('\uFEFF', '').replace('！', '! ').replace('。', '. ').replace('ç','ç').replace('ş','ş').replace('６','6').replace('＠', '@').replace('～', '~')
                    newstring = re.sub(r'([!.]) $', '\1', newstring)
                    if newstring != en_string:
                        print(f"{name} ({lang}) = \"{newstring}\"")
                        done[lang] = newstring
                        if not sect in language_strings[lang]: language_strings[lang][sect] = {}
                        language_strings[lang][sect][name] = newstring
                    else:
                        print(f"{name} ({lang}) = (same as English)")

    # Write a single language entry to the CSV file with narrow, wide, and tall strings
    def write_csv_lang(f, strings, name):
        f.write(',')
        if name in strings['narrow']: f.write('"%s"' % strings['narrow'][name])
        f.write(',')
        if name in strings['wide']: f.write('"%s"' % strings['wide'][name])
        f.write(',')
        if name in strings['tall']: f.write('"%s"' % strings['tall'][name])

    if args.single:
        #
        # Export one large sheet containing all specified languages
        #
        with open("languages.csv", 'w', encoding='utf-8') as f:
            header = ['name']
            for lang in langcodes:
                lname = lang + ' ' + language_name(lang)
                header += [lname, lname + ' (wide)', lname + ' (tall)']
            f.write('"' + '","'.join(header) + '"\n')

            for name in names.keys():
                f.write('"' + name + '"')
                for lang in langcodes: write_csv_lang(f, language_strings[lang], name)
                f.write('\n')
    else:
        #
        # Export a separate sheet for each language
        #
        OUTDIR.mkdir(exist_ok=True)

        for lang in langcodes:
            with open(OUTDIR / f"language_{lang}.csv", 'w', encoding='utf-8') as f:
                lname = lang + ' ' + language_name(lang)
                header = ['name', lname, lname + ' (wide)', lname + ' (tall)']
                f.write('"' + '","'.join(header) + '"\n')

                for name in names.keys():
                    f.write('"' + name + '"')
                    write_csv_lang(f, language_strings[lang], name)
                    f.write('\n')

if __name__ == "__main__":
    # Check for the path to the language files
    if not Path(LANGHOME).is_dir():
        print(f"Error: Couldn't find the '{LANGHOME}' directory.")
        print("Edit LANGHOME or cd to the root of the repo before running.")
        exit(1)

    # Parse and validate all arguments
    parser = argparse.ArgumentParser(description="Export LCD language strings to CSV with optional translation")
    parser.add_argument('-l', '--language', action="append", default=None, help="specify language(s) to translate from canonical English")
    parser.add_argument('-s', '--single', action="store_true", help="output a single spreadsheet (languages.csv)")
    parser.add_argument('-v', '--verbose', action="store_true", help="extra output for debugging")
    parser.add_argument('-n', '--limit', default=0, help="limit the number of exported items")
    parser.add_argument('-t', '--translate', action="store_true", help="use an LLM to translate strings")
    parser.add_argument('-d', '--dothink', action="store_true", help="use thinking if the model supports it")
    parser.add_argument('-m', '--model', default=None, help="override the default LLM model for translation")
    args = parser.parse_args()

    if not args.translate:
        if args.model: print("--model ignored when not translating")
        if args.dothink: print("--dothink ignored when not translating")

    language_export(args)