FreeCAD

1
#!/usr/bin/env python3
2

3
# SPDX-License-Identifier: LGPL-2.1-or-later
4
# ***************************************************************************
5
# *                                                                         *
6
# *   Copyright (c) 2015 Yorik van Havre <yorik@uncreated.net>              *
7
# *   Copyright (c) 2021 Benjamin Nauck <benjamin@nauck.se>                 *
8
# *   Copyright (c) 2021 Mattias Pierre <github@mattiaspierre.com>          *
9
# *                                                                         *
10
# *   This file is part of FreeCAD.                                         *
11
# *                                                                         *
12
# *   FreeCAD is free software: you can redistribute it and/or modify it    *
13
# *   under the terms of the GNU Lesser General Public License as           *
14
# *   published by the Free Software Foundation, either version 2.1 of the  *
15
# *   License, or (at your option) any later version.                       *
16
# *                                                                         *
17
# *   FreeCAD is distributed in the hope that it will be useful, but        *
18
# *   WITHOUT ANY WARRANTY; without even the implied warranty of            *
19
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU      *
20
# *   Lesser General Public License for more details.                       *
21
# *                                                                         *
22
# *   You should have received a copy of the GNU Lesser General Public      *
23
# *   License along with FreeCAD. If not, see                               *
24
# *   <https://www.gnu.org/licenses/>.                                      *
25
# *                                                                         *
26
# ***************************************************************************
27

28
"""
29
This utility offers several commands to interact with the FreeCAD project on
30
crowdin. For it to work, you need a ~/.crowdin-freecad-token file in your
31
user's folder, that contains the API access token that gives access to the
32
crowdin FreeCAD project. The API token can also be specified in the
33
CROWDIN_TOKEN environment variable.
34

35
The CROWDIN_PROJECT_ID environment variable can be used to use this script
36
in other projects.
37

38
Usage:
39

40
    updatecrowdin.py <command> [<arguments>]
41

42
Available commands:
43

44
    gather:              update all ts files found in the source code
45
                         (runs updatets.py)
46
    status:              prints a status of the translations
47
    update / upload:     updates crowdin the current version of .ts files
48
                         found in the source code
49
    build:               builds a new downloadable package on crowdin with all
50
                         translated strings
51
    build-status:        shows the status of the current builds available on
52
                         crowdin
53
    download [build_id]: downloads build specified by 'build_id' or latest if
54
                         build_id is left blank
55
    apply / install:     applies downloaded translations to source code
56
                         (runs updatefromcrowdin.py)
57

58
Example:
59

60
    ./updatecrowdin.py update
61

62
Setting the project name adhoc:
63

64
    CROWDIN_PROJECT_ID=some_project ./updatecrowdin.py update
65
"""
66

67
# See crowdin API docs at https://crowdin.com/page/api
68

69
import concurrent.futures
70
import glob
71
import json
72
import os
73
import sys
74
import shutil
75
import subprocess
76
import tempfile
77
import zipfile
78
import re
79
from collections import namedtuple
80
from functools import lru_cache
81
from os.path import basename, splitext
82
from urllib.parse import quote_plus
83
from urllib.request import Request
84
from urllib.request import urlopen
85
from urllib.request import urlretrieve
86

87
try:
88
    from PySide6 import QtCore
89
except ImportError:
90
    from PySide2 import QtCore
91

92
TsFile = namedtuple("TsFile", ["filename", "src_path"])
93

94
LEGACY_NAMING_MAP = {"Draft.ts": "draft.ts"}
95

96
# Locations that require QM file generation (predominantly Python workbenches)
97
GENERATE_QM = {
98
    "AddonManager",
99
    "Arch",
100
    "Cloud",
101
    "Draft",
102
    "Inspection",
103
    "OpenSCAD",
104
    "Tux",
105
    "Help",
106
}
107

108
# locations list contains Module name, relative path to translation folder and relative path to qrc file
109

110
locations = [
111
    [
112
        "AddonManager",
113
        "../Mod/AddonManager/Resources/translations",
114
        "../Mod/AddonManager/Resources/AddonManager.qrc",
115
    ],
116
    ["App", "../App/Resources/translations", "../App/Resources/App.qrc"],
117
    ["Arch", "../Mod/BIM/Resources/translations", "../Mod/BIM/Resources/Arch.qrc"],
118
    [
119
        "Assembly",
120
        "../Mod/Assembly/Gui/Resources/translations",
121
        "../Mod/Assembly/Gui/Resources/Assembly.qrc",
122
    ],
123
    [
124
        "draft",
125
        "../Mod/Draft/Resources/translations",
126
        "../Mod/Draft/Resources/Draft.qrc",
127
    ],
128
    ["Base", "../Base/Resources/translations", "../Base/Resources/Base.qrc"],
129
    [
130
        "Drawing",
131
        "../Mod/Drawing/Gui/Resources/translations",
132
        "../Mod/Drawing/Gui/Resources/Drawing.qrc",
133
    ],
134
    [
135
        "Fem",
136
        "../Mod/Fem/Gui/Resources/translations",
137
        "../Mod/Fem/Gui/Resources/Fem.qrc",
138
    ],
139
    ["FreeCAD", "../Gui/Language", "../Gui/Language/translation.qrc"],
140
    ["Help", "../Mod/Help/Resources/translations", "../Mod/Help/Resources/Help.qrc"],
141
    [
142
        "Inspection",
143
        "../Mod/Inspection/Gui/Resources/translations",
144
        "../Mod/Inspection/Gui/Resources/Inspection.qrc",
145
    ],
146
    [
147
        "Material",
148
        "../Mod/Material/Gui/Resources/translations",
149
        "../Mod/Material/Gui/Resources/Material.qrc",
150
    ],
151
    [
152
        "Mesh",
153
        "../Mod/Mesh/Gui/Resources/translations",
154
        "../Mod/Mesh/Gui/Resources/Mesh.qrc",
155
    ],
156
    [
157
        "MeshPart",
158
        "../Mod/MeshPart/Gui/Resources/translations",
159
        "../Mod/MeshPart/Gui/Resources/MeshPart.qrc",
160
    ],
161
    [
162
        "OpenSCAD",
163
        "../Mod/OpenSCAD/Resources/translations",
164
        "../Mod/OpenSCAD/Resources/OpenSCAD.qrc",
165
    ],
166
    [
167
        "Part",
168
        "../Mod/Part/Gui/Resources/translations",
169
        "../Mod/Part/Gui/Resources/Part.qrc",
170
    ],
171
    [
172
        "PartDesign",
173
        "../Mod/PartDesign/Gui/Resources/translations",
174
        "../Mod/PartDesign/Gui/Resources/PartDesign.qrc",
175
    ],
176
    [
177
        "CAM",
178
        "../Mod/CAM/Gui/Resources/translations",
179
        "../Mod/CAM/Gui/Resources/CAM.qrc",
180
    ],
181
    [
182
        "Points",
183
        "../Mod/Points/Gui/Resources/translations",
184
        "../Mod/Points/Gui/Resources/Points.qrc",
185
    ],
186
    [
187
        "ReverseEngineering",
188
        "../Mod/ReverseEngineering/Gui/Resources/translations",
189
        "../Mod/ReverseEngineering/Gui/Resources/ReverseEngineering.qrc",
190
    ],
191
    [
192
        "Robot",
193
        "../Mod/Robot/Gui/Resources/translations",
194
        "../Mod/Robot/Gui/Resources/Robot.qrc",
195
    ],
196
    [
197
        "Sketcher",
198
        "../Mod/Sketcher/Gui/Resources/translations",
199
        "../Mod/Sketcher/Gui/Resources/Sketcher.qrc",
200
    ],
201
    [
202
        "Spreadsheet",
203
        "../Mod/Spreadsheet/Gui/Resources/translations",
204
        "../Mod/Spreadsheet/Gui/Resources/Spreadsheet.qrc",
205
    ],
206
    [
207
        "StartPage",
208
        "../Mod/Start/Gui/Resources/translations",
209
        "../Mod/Start/Gui/Resources/Start.qrc",
210
    ],
211
    [
212
        "Test",
213
        "../Mod/Test/Gui/Resources/translations",
214
        "../Mod/Test/Gui/Resources/Test.qrc",
215
    ],
216
    [
217
        "TechDraw",
218
        "../Mod/TechDraw/Gui/Resources/translations",
219
        "../Mod/TechDraw/Gui/Resources/TechDraw.qrc",
220
    ],
221
    ["Tux", "../Mod/Tux/Resources/translations", "../Mod/Tux/Resources/Tux.qrc"],
222
]
223

224
THRESHOLD = 25  # how many % must be translated for the translation to be included in FreeCAD
225

226

227
class CrowdinUpdater:
228

229
    BASE_URL = "https://api.crowdin.com/api/v2"
230

231
    def __init__(self, token, project_identifier, multithread=True):
232
        self.token = token
233
        self.project_identifier = project_identifier
234
        self.multithread = multithread
235

236
    @lru_cache()
237
    def _get_project_id(self):
238
        url = f"{self.BASE_URL}/projects/"
239
        response = self._make_api_req(url)
240

241
        for project in [p["data"] for p in response]:
242
            if project["identifier"] == project_identifier:
243
                return project["id"]
244

245
        raise Exception("No project identifier found!")
246

247
    def _make_project_api_req(self, project_path, *args, **kwargs):
248
        url = f"{self.BASE_URL}/projects/{self._get_project_id()}{project_path}"
249
        return self._make_api_req(url=url, *args, **kwargs)
250

251
    def _make_api_req(self, url, extra_headers={}, method="GET", data=None):
252
        headers = {"Authorization": "Bearer " + load_token(), **extra_headers}
253

254
        if type(data) is dict:
255
            headers["Content-Type"] = "application/json"
256
            data = json.dumps(data).encode("utf-8")
257

258
        request = Request(url, headers=headers, method=method, data=data)
259
        return json.loads(urlopen(request).read())["data"]
260

261
    def _get_files_info(self):
262
        files = self._make_project_api_req("/files?limit=250")
263
        return {f["data"]["path"].strip("/"): str(f["data"]["id"]) for f in files}
264

265
    def _add_storage(self, filename, fp):
266
        response = self._make_api_req(
267
            f"{self.BASE_URL}/storages",
268
            data=fp,
269
            method="POST",
270
            extra_headers={
271
                "Crowdin-API-FileName": filename,
272
                "Content-Type": "application/octet-stream",
273
            },
274
        )
275
        return response["id"]
276

277
    def _update_file(self, project_id, ts_file, files_info):
278
        filename = quote_plus(ts_file.filename)
279

280
        with open(ts_file.src_path, "rb") as fp:
281
            storage_id = self._add_storage(filename, fp)
282

283
        if filename in files_info:
284
            file_id = files_info[filename]
285
            self._make_project_api_req(
286
                f"/files/{file_id}",
287
                method="PUT",
288
                data={
289
                    "storageId": storage_id,
290
                    "updateOption": "keep_translations_and_approvals",
291
                },
292
            )
293
            print(f"{filename} updated")
294
        else:
295
            self._make_project_api_req("/files", data={"storageId": storage_id, "name": filename})
296
            print(f"{filename} uploaded")
297

298
    def status(self):
299
        response = self._make_project_api_req("/languages/progress?limit=100")
300
        return [item["data"] for item in response]
301

302
    def download(self, build_id):
303
        filename = f"{self.project_identifier}.zip"
304
        response = self._make_project_api_req(f"/translations/builds/{build_id}/download")
305
        urlretrieve(response["url"], filename)
306
        print("download of " + filename + " complete")
307

308
    def build(self):
309
        self._make_project_api_req("/translations/builds", data={}, method="POST")
310

311
    def build_status(self):
312
        response = self._make_project_api_req("/translations/builds")
313
        return [item["data"] for item in response]
314

315
    def update(self, ts_files):
316
        files_info = self._get_files_info()
317
        futures = []
318

319
        with concurrent.futures.ThreadPoolExecutor() as executor:
320
            for ts_file in ts_files:
321
                if self.multithread:
322
                    future = executor.submit(
323
                        self._update_file, self.project_identifier, ts_file, files_info
324
                    )
325
                    futures.append(future)
326
                else:
327
                    self._update_file(self.project_identifier, ts_file, files_info)
328

329
        # This blocks until all futures are complete and will also throw any exception
330
        for future in futures:
331
            future.result()
332

333

334
def load_token():
335
    # load API token stored in ~/.crowdin-freecad-token
336
    config_file = os.path.expanduser("~") + os.sep + ".crowdin-freecad-token"
337
    if os.path.exists(config_file):
338
        with open(config_file) as file:
339
            return file.read().strip()
340
    return None
341

342

343
def updateqrc(qrcpath, lncode):
344

345
    "updates a qrc file with the given translation entry"
346

347
    # print("opening " + qrcpath + "...")
348

349
    # getting qrc file contents
350
    if not os.path.exists(qrcpath):
351
        print("ERROR: Resource file " + qrcpath + " doesn't exist")
352
        sys.exit()
353
    f = open(qrcpath, "r")
354
    resources = []
355
    for l in f.readlines():
356
        resources.append(l)
357
    f.close()
358

359
    # checking for existing entry
360
    name = "_" + lncode + ".qm"
361
    for r in resources:
362
        if name in r:
363
            # print("language already exists in qrc file")
364
            return
365

366
    # find the latest qm line
367
    pos = None
368
    for i in range(len(resources)):
369
        if ".qm" in resources[i]:
370
            pos = i
371
    if pos is None:
372
        print("No existing .qm file in this resource. Appending to the end position")
373
        for i in range(len(resources)):
374
            if "</qresource>" in resources[i]:
375
                pos = i - 1
376
    if pos is None:
377
        print("ERROR: couldn't add qm files to this resource: " + qrcpath)
378
        sys.exit()
379

380
    # inserting new entry just after the last one
381
    line = resources[pos]
382
    if ".qm" in line:
383
        line = re.sub(r"_.*\.qm", "_" + lncode + ".qm", line)
384
    else:
385
        modname = os.path.splitext(os.path.basename(qrcpath))[0]
386
        line = "        <file>translations/" + modname + "_" + lncode + ".qm</file>\n"
387
        # print "ERROR: no existing qm entry in this resource: Please add one manually " + qrcpath
388
        # sys.exit()
389
    # print("inserting line: ",line)
390
    resources.insert(pos + 1, line)
391

392
    # writing the file
393
    f = open(qrcpath, "w")
394
    for r in resources:
395
        f.write(r)
396
    f.close()
397
    print("successfully updated ", qrcpath)
398

399

400
def updateTranslatorCpp(lncode):
401

402
    "updates the Translator.cpp file with the given translation entry"
403

404
    cppfile = os.path.join(os.path.dirname(__file__), "..", "Gui", "Language", "Translator.cpp")
405
    l = QtCore.QLocale(lncode)
406
    lnname = QtCore.QLocale.languageToString(l.language())
407

408
    # read file contents
409
    f = open(cppfile, "r")
410
    cppcode = []
411
    for l in f.readlines():
412
        cppcode.append(l)
413
    f.close()
414

415
    # checking for existing entry
416
    lastentry = 0
417
    for i, l in enumerate(cppcode):
418
        if l.startswith("    d->mapLanguageTopLevelDomain[QT_TR_NOOP("):
419
            lastentry = i
420
            if '"' + lncode + '"' in l:
421
                # print(lnname+" ("+lncode+") already exists in Translator.cpp")
422
                return
423

424
    # find the position to insert
425
    pos = lastentry + 1
426
    if pos == 1:
427
        print("ERROR: couldn't update Translator.cpp")
428
        sys.exit()
429

430
    # inserting new entry just before the above line
431
    line = '    d->mapLanguageTopLevelDomain[QT_TR_NOOP("' + lnname + '")] = "' + lncode + '";\n'
432
    cppcode.insert(pos, line)
433
    print(lnname + " (" + lncode + ") added Translator.cpp")
434

435
    # writing the file
436
    f = open(cppfile, "w")
437
    for r in cppcode:
438
        f.write(r)
439
    f.close()
440

441

442
def doFile(tsfilepath, targetpath, lncode, qrcpath):
443

444
    "updates a single ts file, and creates a corresponding qm file"
445

446
    basename = os.path.basename(tsfilepath)[:-3]
447
    # filename fixes
448
    if basename + ".ts" in LEGACY_NAMING_MAP.values():
449
        basename = list(LEGACY_NAMING_MAP)[
450
            list(LEGACY_NAMING_MAP.values()).index(basename + ".ts")
451
        ][:-3]
452
    newname = basename + "_" + lncode + ".ts"
453
    newpath = targetpath + os.sep + newname
454
    if not os.path.exists(tsfilepath):
455
        # If this language code does not exist for the given TS file, bail out
456
        return
457
    shutil.copyfile(tsfilepath, newpath)
458
    if basename in GENERATE_QM:
459
        # print("generating qm files for",newpath,"...")
460
        try:
461
            subprocess.run(
462
                [
463
                    "lrelease",
464
                    newpath,
465
                ],
466
                timeout=5,
467
            )
468
        except Exception as e:
469
            print(e)
470
        newqm = targetpath + os.sep + basename + "_" + lncode + ".qm"
471
        if not os.path.exists(newqm):
472
            print("ERROR: failed to create " + newqm + ", aborting")
473
            sys.exit()
474
        updateqrc(qrcpath, lncode)
475

476

477
def doLanguage(lncode):
478

479
    "treats a single language"
480

481
    if lncode == "en":
482
        # never treat "english" translation... For now :)
483
        return
484
    prefix = ""
485
    suffix = ""
486
    if os.name == "posix":
487
        prefix = "\033[;32m"
488
        suffix = "\033[0m"
489
    print("Updating files for " + prefix + lncode + suffix + "...", end="")
490
    for target in locations:
491
        basefilepath = os.path.join(tempfolder, lncode, target[0] + ".ts")
492
        targetpath = os.path.abspath(target[1])
493
        qrcpath = os.path.abspath(target[2])
494
        doFile(basefilepath, targetpath, lncode, qrcpath)
495
    print(" done")
496

497

498
def applyTranslations(languages):
499

500
    global tempfolder
501
    currentfolder = os.getcwd()
502
    tempfolder = tempfile.mkdtemp()
503
    print("creating temp folder " + tempfolder)
504
    src = os.path.join(currentfolder, "freecad.zip")
505
    dst = os.path.join(tempfolder, "freecad.zip")
506
    if not os.path.exists(src):
507
        print('freecad.zip file not found! Aborting. Run "download" command before this one.')
508
        sys.exit()
509
    shutil.copyfile(src, dst)
510
    os.chdir(tempfolder)
511
    zfile = zipfile.ZipFile("freecad.zip")
512
    print("extracting freecad.zip...")
513
    zfile.extractall()
514
    os.chdir(currentfolder)
515
    for ln in languages:
516
        if not os.path.exists(os.path.join(tempfolder, ln)):
517
            print("ERROR: language path for " + ln + " not found!")
518
        else:
519
            doLanguage(ln)
520

521

522
if __name__ == "__main__":
523
    command = None
524

525
    args = sys.argv[1:]
526
    if args:
527
        command = args[0]
528

529
    token = os.environ.get("CROWDIN_TOKEN", load_token())
530
    if command and not token:
531
        print("Token not found")
532
        sys.exit()
533

534
    project_identifier = os.environ.get("CROWDIN_PROJECT_ID")
535
    if not project_identifier:
536
        project_identifier = "freecad"
537
        # print('CROWDIN_PROJECT_ID env var must be set')
538
        # sys.exit()
539

540
    updater = CrowdinUpdater(token, project_identifier)
541

542
    if command == "status":
543
        status = updater.status()
544
        status = sorted(status, key=lambda item: item["translationProgress"], reverse=True)
545
        print(
546
            len([item for item in status if item["translationProgress"] > THRESHOLD]),
547
            " languages with status > " + str(THRESHOLD) + "%:",
548
        )
549
        print("    ")
550
        sep = False
551
        prefix = ""
552
        suffix = ""
553
        if os.name == "posix":
554
            prefix = "\033[;32m"
555
            suffix = "\033[0m"
556
        for item in status:
557
            if item["translationProgress"] > 0:
558
                if (item["translationProgress"] < THRESHOLD) and (not sep):
559
                    print("    ")
560
                    print("Other languages:")
561
                    print("    ")
562
                    sep = True
563
                print(
564
                    prefix
565
                    + item["languageId"]
566
                    + suffix
567
                    + " "
568
                    + str(item["translationProgress"])
569
                    + "% ("
570
                    + str(item["approvalProgress"])
571
                    + "% approved)"
572
                )
573
                # print(f"  translation progress: {item['translationProgress']}%")
574
                # print(f"  approval progress:    {item['approvalProgress']}%")
575

576
    elif command == "build-status":
577
        for item in updater.build_status():
578
            print(f"  id: {item['id']} progress: {item['progress']}% status: {item['status']}")
579

580
    elif command == "build":
581
        updater.build()
582

583
    elif command == "download":
584
        if len(args) == 2:
585
            updater.download(args[1])
586
        else:
587
            stat = updater.build_status()
588
            if not stat:
589
                print("no builds found")
590
            elif len(stat) == 1:
591
                updater.download(stat[0]["id"])
592
            else:
593
                print("available builds:")
594
                for item in stat:
595
                    print(
596
                        f"  id: {item['id']} progress: {item['progress']}% status: {item['status']}"
597
                    )
598
                print("please specify a build id")
599

600
    elif command in ["update", "upload"]:
601
        # Find all ts files. However, this contains the lang-specific files too. Let's drop those
602
        all_ts_files = glob.glob("../**/*.ts", recursive=True)
603
        # Remove the file extensions
604
        ts_files_wo_ext = [splitext(f)[0] for f in all_ts_files]
605
        # Filter out any file that has another file as a substring. E.g. Draft is a substring of Draft_en
606
        main_ts_files = list(
607
            filter(
608
                lambda f: not [a for a in ts_files_wo_ext if a in f and f != a],
609
                ts_files_wo_ext,
610
            )
611
        )
612
        # Create tuples to map Crowdin name with local path name
613
        names_and_path = [(f"{basename(f)}.ts", f"{f}.ts") for f in main_ts_files]
614
        # Accommodate for legacy naming
615
        ts_files = [
616
            TsFile(LEGACY_NAMING_MAP[a] if a in LEGACY_NAMING_MAP else a, b)
617
            for (a, b) in names_and_path
618
        ]
619
        updater.update(ts_files)
620

621
    elif command in ["apply", "install"]:
622
        print("retrieving list of languages...")
623
        status = updater.status()
624
        status = sorted(status, key=lambda item: item["translationProgress"], reverse=True)
625
        languages = [
626
            item["languageId"] for item in status if item["translationProgress"] > THRESHOLD
627
        ]
628
        applyTranslations(languages)
629
        print("Updating Translator.cpp...")
630
        for ln in languages:
631
            updateTranslatorCpp(ln)
632

633
    elif command == "updateTranslator":
634
        print("retrieving list of languages...")
635
        status = updater.status()
636
        status = sorted(status, key=lambda item: item["translationProgress"], reverse=True)
637
        languages = [
638
            item["languageId"] for item in status if item["translationProgress"] > THRESHOLD
639
        ]
640
        print("Updating Translator.cpp...")
641
        for ln in languages:
642
            updateTranslatorCpp(ln)
643

644
    elif command == "gather":
645
        import updatets
646

647
        updatets.main()
648

649
    else:
650
        print(__doc__)
651