FreeCAD

1
#!/usr/bin/env python3
2

3
# SPDX-License-Identifier: LGPL-2.1-or-later
4
# ***************************************************************************
5
# *                                                                         *
6
# *   Copyright (c) 2015 Yorik van Havre <yorik@uncreated.net>              *
7
# *   Copyright (c) 2021 Benjamin Nauck <benjamin@nauck.se>                 *
8
# *   Copyright (c) 2021 Mattias Pierre <github@mattiaspierre.com>          *
9
# *                                                                         *
10
# *   This file is part of FreeCAD.                                         *
11
# *                                                                         *
12
# *   FreeCAD is free software: you can redistribute it and/or modify it    *
13
# *   under the terms of the GNU Lesser General Public License as           *
14
# *   published by the Free Software Foundation, either version 2.1 of the  *
15
# *   License, or (at your option) any later version.                       *
16
# *                                                                         *
17
# *   FreeCAD is distributed in the hope that it will be useful, but        *
18
# *   WITHOUT ANY WARRANTY; without even the implied warranty of            *
19
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU      *
20
# *   Lesser General Public License for more details.                       *
21
# *                                                                         *
22
# *   You should have received a copy of the GNU Lesser General Public      *
23
# *   License along with FreeCAD. If not, see                               *
24
# *   <https://www.gnu.org/licenses/>.                                      *
25
# *                                                                         *
26
# ***************************************************************************
27

28
"""
29
This utility offers several commands to interact with the FreeCAD project on
30
crowdin. For it to work, you need a ~/.crowdin-freecad-token file in your
31
user's folder, that contains the API access token that gives access to the
32
crowdin FreeCAD project. The API token can also be specified in the
33
CROWDIN_TOKEN environment variable.
34

35
The CROWDIN_PROJECT_ID environment variable can be used to use this script
36
in other projects.
37

38
Usage:
39

40
    updatecrowdin.py <command> [<arguments>]
41

42
Available commands:
43

44
    gather:              update all ts files found in the source code
45
                         (runs updatets.py)
46
    status:              prints a status of the translations
47
    update / upload:     updates crowdin the current version of .ts files
48
                         found in the source code
49
    build:               builds a new downloadable package on crowdin with all
50
                         translated strings
51
    build-status:        shows the status of the current builds available on
52
                         crowdin
53
    download [build_id]: downloads build specified by 'build_id' or latest if
54
                         build_id is left blank
55
    apply / install:     applies downloaded translations to source code
56
                         (runs updatefromcrowdin.py)
57

58
Example:
59

60
    ./updatecrowdin.py update
61

62
Setting the project name adhoc:
63

64
    CROWDIN_PROJECT_ID=some_project ./updatecrowdin.py update
65
"""
66

67
# See crowdin API docs at https://crowdin.com/page/api
68

69
import concurrent.futures
70
import glob
71
import json
72
import os
73
import sys
74
import shutil
75
import subprocess
76
import tempfile
77
import zipfile
78
import re
79
from collections import namedtuple
80
from functools import lru_cache
81
from os.path import basename, splitext
82
from urllib.parse import quote_plus
83
from urllib.request import Request
84
from urllib.request import urlopen
85
from urllib.request import urlretrieve
86
from PySide2 import QtCore
87

88
TsFile = namedtuple("TsFile", ["filename", "src_path"])
89

90
LEGACY_NAMING_MAP = {"Draft.ts": "draft.ts"}
91

92
# Locations that require QM file generation (predominantly Python workbenches)
93
GENERATE_QM = {
94
    "AddonManager",
95
    "Arch",
96
    "Cloud",
97
    "Draft",
98
    "Inspection",
99
    "OpenSCAD",
100
    "Tux",
101
    "Help",
102
}
103

104
# locations list contains Module name, relative path to translation folder and relative path to qrc file
105

106
locations = [
107
    [
108
        "AddonManager",
109
        "../Mod/AddonManager/Resources/translations",
110
        "../Mod/AddonManager/Resources/AddonManager.qrc",
111
    ],
112
    ["App", "../App/Resources/translations", "../App/Resources/App.qrc"],
113
    ["Arch", "../Mod/Arch/Resources/translations", "../Mod/Arch/Resources/Arch.qrc"],
114
    [
115
        "Assembly",
116
        "../Mod/Assembly/Gui/Resources/translations",
117
        "../Mod/Assembly/Gui/Resources/Assembly.qrc",
118
    ],
119
    [
120
        "draft",
121
        "../Mod/Draft/Resources/translations",
122
        "../Mod/Draft/Resources/Draft.qrc",
123
    ],
124
    ["Base", "../Base/Resources/translations", "../Base/Resources/Base.qrc"],
125
    [
126
        "Drawing",
127
        "../Mod/Drawing/Gui/Resources/translations",
128
        "../Mod/Drawing/Gui/Resources/Drawing.qrc",
129
    ],
130
    [
131
        "Fem",
132
        "../Mod/Fem/Gui/Resources/translations",
133
        "../Mod/Fem/Gui/Resources/Fem.qrc",
134
    ],
135
    ["FreeCAD", "../Gui/Language", "../Gui/Language/translation.qrc"],
136
    ["Help", "../Mod/Help/Resources/translations", "../Mod/Help/Resources/Help.qrc"],
137
    [
138
        "Inspection",
139
        "../Mod/Inspection/Gui/Resources/translations",
140
        "../Mod/Inspection/Gui/Resources/Inspection.qrc",
141
    ],
142
    [
143
        "Material",
144
        "../Mod/Material/Gui/Resources/translations",
145
        "../Mod/Material/Gui/Resources/Material.qrc",
146
    ],
147
    [
148
        "Mesh",
149
        "../Mod/Mesh/Gui/Resources/translations",
150
        "../Mod/Mesh/Gui/Resources/Mesh.qrc",
151
    ],
152
    [
153
        "MeshPart",
154
        "../Mod/MeshPart/Gui/Resources/translations",
155
        "../Mod/MeshPart/Gui/Resources/MeshPart.qrc",
156
    ],
157
    [
158
        "OpenSCAD",
159
        "../Mod/OpenSCAD/Resources/translations",
160
        "../Mod/OpenSCAD/Resources/OpenSCAD.qrc",
161
    ],
162
    [
163
        "Part",
164
        "../Mod/Part/Gui/Resources/translations",
165
        "../Mod/Part/Gui/Resources/Part.qrc",
166
    ],
167
    [
168
        "PartDesign",
169
        "../Mod/PartDesign/Gui/Resources/translations",
170
        "../Mod/PartDesign/Gui/Resources/PartDesign.qrc",
171
    ],
172
    [
173
        "CAM",
174
        "../Mod/CAM/Gui/Resources/translations",
175
        "../Mod/CAM/Gui/Resources/CAM.qrc",
176
    ],
177
    [
178
        "Points",
179
        "../Mod/Points/Gui/Resources/translations",
180
        "../Mod/Points/Gui/Resources/Points.qrc",
181
    ],
182
    [
183
        "ReverseEngineering",
184
        "../Mod/ReverseEngineering/Gui/Resources/translations",
185
        "../Mod/ReverseEngineering/Gui/Resources/ReverseEngineering.qrc",
186
    ],
187
    [
188
        "Robot",
189
        "../Mod/Robot/Gui/Resources/translations",
190
        "../Mod/Robot/Gui/Resources/Robot.qrc",
191
    ],
192
    [
193
        "Sketcher",
194
        "../Mod/Sketcher/Gui/Resources/translations",
195
        "../Mod/Sketcher/Gui/Resources/Sketcher.qrc",
196
    ],
197
    [
198
        "Spreadsheet",
199
        "../Mod/Spreadsheet/Gui/Resources/translations",
200
        "../Mod/Spreadsheet/Gui/Resources/Spreadsheet.qrc",
201
    ],
202
    [
203
        "StartPage",
204
        "../Mod/Start/Gui/Resources/translations",
205
        "../Mod/Start/Gui/Resources/Start.qrc",
206
    ],
207
    [
208
        "Test",
209
        "../Mod/Test/Gui/Resources/translations",
210
        "../Mod/Test/Gui/Resources/Test.qrc",
211
    ],
212
    [
213
        "TechDraw",
214
        "../Mod/TechDraw/Gui/Resources/translations",
215
        "../Mod/TechDraw/Gui/Resources/TechDraw.qrc",
216
    ],
217
    ["Tux", "../Mod/Tux/Resources/translations", "../Mod/Tux/Resources/Tux.qrc"],
218
    [
219
        "Web",
220
        "../Mod/Web/Gui/Resources/translations",
221
        "../Mod/Web/Gui/Resources/Web.qrc",
222
    ],
223
]
224

225
THRESHOLD = 25  # how many % must be translated for the translation to be included in FreeCAD
226

227

228
class CrowdinUpdater:
229

230
    BASE_URL = "https://api.crowdin.com/api/v2"
231

232
    def __init__(self, token, project_identifier, multithread=True):
233
        self.token = token
234
        self.project_identifier = project_identifier
235
        self.multithread = multithread
236

237
    @lru_cache()
238
    def _get_project_id(self):
239
        url = f"{self.BASE_URL}/projects/"
240
        response = self._make_api_req(url)
241

242
        for project in [p["data"] for p in response]:
243
            if project["identifier"] == project_identifier:
244
                return project["id"]
245

246
        raise Exception("No project identifier found!")
247

248
    def _make_project_api_req(self, project_path, *args, **kwargs):
249
        url = f"{self.BASE_URL}/projects/{self._get_project_id()}{project_path}"
250
        return self._make_api_req(url=url, *args, **kwargs)
251

252
    def _make_api_req(self, url, extra_headers={}, method="GET", data=None):
253
        headers = {"Authorization": "Bearer " + load_token(), **extra_headers}
254

255
        if type(data) is dict:
256
            headers["Content-Type"] = "application/json"
257
            data = json.dumps(data).encode("utf-8")
258

259
        request = Request(url, headers=headers, method=method, data=data)
260
        return json.loads(urlopen(request).read())["data"]
261

262
    def _get_files_info(self):
263
        files = self._make_project_api_req("/files?limit=250")
264
        return {f["data"]["path"].strip("/"): str(f["data"]["id"]) for f in files}
265

266
    def _add_storage(self, filename, fp):
267
        response = self._make_api_req(
268
            f"{self.BASE_URL}/storages",
269
            data=fp,
270
            method="POST",
271
            extra_headers={
272
                "Crowdin-API-FileName": filename,
273
                "Content-Type": "application/octet-stream",
274
            },
275
        )
276
        return response["id"]
277

278
    def _update_file(self, project_id, ts_file, files_info):
279
        filename = quote_plus(ts_file.filename)
280

281
        with open(ts_file.src_path, "rb") as fp:
282
            storage_id = self._add_storage(filename, fp)
283

284
        if filename in files_info:
285
            file_id = files_info[filename]
286
            self._make_project_api_req(
287
                f"/files/{file_id}",
288
                method="PUT",
289
                data={
290
                    "storageId": storage_id,
291
                    "updateOption": "keep_translations_and_approvals",
292
                },
293
            )
294
            print(f"{filename} updated")
295
        else:
296
            self._make_project_api_req("/files", data={"storageId": storage_id, "name": filename})
297
            print(f"{filename} uploaded")
298

299
    def status(self):
300
        response = self._make_project_api_req("/languages/progress?limit=100")
301
        return [item["data"] for item in response]
302

303
    def download(self, build_id):
304
        filename = f"{self.project_identifier}.zip"
305
        response = self._make_project_api_req(f"/translations/builds/{build_id}/download")
306
        urlretrieve(response["url"], filename)
307
        print("download of " + filename + " complete")
308

309
    def build(self):
310
        self._make_project_api_req("/translations/builds", data={}, method="POST")
311

312
    def build_status(self):
313
        response = self._make_project_api_req("/translations/builds")
314
        return [item["data"] for item in response]
315

316
    def update(self, ts_files):
317
        files_info = self._get_files_info()
318
        futures = []
319

320
        with concurrent.futures.ThreadPoolExecutor() as executor:
321
            for ts_file in ts_files:
322
                if self.multithread:
323
                    future = executor.submit(
324
                        self._update_file, self.project_identifier, ts_file, files_info
325
                    )
326
                    futures.append(future)
327
                else:
328
                    self._update_file(self.project_identifier, ts_file, files_info)
329

330
        # This blocks until all futures are complete and will also throw any exception
331
        for future in futures:
332
            future.result()
333

334

335
def load_token():
336
    # load API token stored in ~/.crowdin-freecad-token
337
    config_file = os.path.expanduser("~") + os.sep + ".crowdin-freecad-token"
338
    if os.path.exists(config_file):
339
        with open(config_file) as file:
340
            return file.read().strip()
341
    return None
342

343

344
def updateqrc(qrcpath, lncode):
345

346
    "updates a qrc file with the given translation entry"
347

348
    # print("opening " + qrcpath + "...")
349

350
    # getting qrc file contents
351
    if not os.path.exists(qrcpath):
352
        print("ERROR: Resource file " + qrcpath + " doesn't exist")
353
        sys.exit()
354
    f = open(qrcpath, "r")
355
    resources = []
356
    for l in f.readlines():
357
        resources.append(l)
358
    f.close()
359

360
    # checking for existing entry
361
    name = "_" + lncode + ".qm"
362
    for r in resources:
363
        if name in r:
364
            # print("language already exists in qrc file")
365
            return
366

367
    # find the latest qm line
368
    pos = None
369
    for i in range(len(resources)):
370
        if ".qm" in resources[i]:
371
            pos = i
372
    if pos is None:
373
        print("No existing .qm file in this resource. Appending to the end position")
374
        for i in range(len(resources)):
375
            if "</qresource>" in resources[i]:
376
                pos = i - 1
377
    if pos is None:
378
        print("ERROR: couldn't add qm files to this resource: " + qrcpath)
379
        sys.exit()
380

381
    # inserting new entry just after the last one
382
    line = resources[pos]
383
    if ".qm" in line:
384
        line = re.sub("_.*\.qm", "_" + lncode + ".qm", line)
385
    else:
386
        modname = os.path.splitext(os.path.basename(qrcpath))[0]
387
        line = "        <file>translations/" + modname + "_" + lncode + ".qm</file>\n"
388
        # print "ERROR: no existing qm entry in this resource: Please add one manually " + qrcpath
389
        # sys.exit()
390
    # print("inserting line: ",line)
391
    resources.insert(pos + 1, line)
392

393
    # writing the file
394
    f = open(qrcpath, "w")
395
    for r in resources:
396
        f.write(r)
397
    f.close()
398
    print("successfully updated ", qrcpath)
399

400

401
def updateTranslatorCpp(lncode):
402

403
    "updates the Translator.cpp file with the given translation entry"
404

405
    cppfile = os.path.join(os.path.dirname(__file__), "..", "Gui", "Language", "Translator.cpp")
406
    l = QtCore.QLocale(lncode)
407
    lnname = l.languageToString(l.language())
408

409
    # read file contents
410
    f = open(cppfile, "r")
411
    cppcode = []
412
    for l in f.readlines():
413
        cppcode.append(l)
414
    f.close()
415

416
    # checking for existing entry
417
    lastentry = 0
418
    for i, l in enumerate(cppcode):
419
        if l.startswith("    d->mapLanguageTopLevelDomain[QT_TR_NOOP("):
420
            lastentry = i
421
            if '"' + lncode + '"' in l:
422
                # print(lnname+" ("+lncode+") already exists in Translator.cpp")
423
                return
424

425
    # find the position to insert
426
    pos = lastentry + 1
427
    if pos == 1:
428
        print("ERROR: couldn't update Translator.cpp")
429
        sys.exit()
430

431
    # inserting new entry just before the above line
432
    line = '    d->mapLanguageTopLevelDomain[QT_TR_NOOP("' + lnname + '")] = "' + lncode + '";\n'
433
    cppcode.insert(pos, line)
434
    print(lnname + " (" + lncode + ") added Translator.cpp")
435

436
    # writing the file
437
    f = open(cppfile, "w")
438
    for r in cppcode:
439
        f.write(r)
440
    f.close()
441

442

443
def doFile(tsfilepath, targetpath, lncode, qrcpath):
444

445
    "updates a single ts file, and creates a corresponding qm file"
446

447
    basename = os.path.basename(tsfilepath)[:-3]
448
    # filename fixes
449
    if basename + ".ts" in LEGACY_NAMING_MAP.values():
450
        basename = list(LEGACY_NAMING_MAP)[
451
            list(LEGACY_NAMING_MAP.values()).index(basename + ".ts")
452
        ][:-3]
453
    newname = basename + "_" + lncode + ".ts"
454
    newpath = targetpath + os.sep + newname
455
    if not os.path.exists(newpath):
456
        # If this language code does not exist for the given TS file, bail out
457
        return
458
    shutil.copyfile(tsfilepath, newpath)
459
    if basename in GENERATE_QM:
460
        print(f"Generating QM for {basename}")
461
        try:
462
            subprocess.run(
463
                [
464
                    "lrelease",
465
                    newpath,
466
                ],
467
                timeout=5,
468
            )
469
        except Exception as e:
470
            print(e)
471
        newqm = targetpath + os.sep + basename + "_" + lncode + ".qm"
472
        if not os.path.exists(newqm):
473
            print("ERROR: failed to create " + newqm + ", aborting")
474
            sys.exit()
475
        updateqrc(qrcpath, lncode)
476

477

478
def doLanguage(lncode):
479

480
    "treats a single language"
481

482
    if lncode == "en":
483
        # never treat "english" translation... For now :)
484
        return
485
    prefix = ""
486
    suffix = ""
487
    if os.name == "posix":
488
        prefix = "\033[;32m"
489
        suffix = "\033[0m"
490
    print("Updating files for " + prefix + lncode + suffix + "...", end="")
491
    for target in locations:
492
        basefilepath = os.path.join(tempfolder, lncode, target[0] + ".ts")
493
        targetpath = os.path.abspath(target[1])
494
        qrcpath = os.path.abspath(target[2])
495
        doFile(basefilepath, targetpath, lncode, qrcpath)
496
    print(" done")
497

498

499
def applyTranslations(languages):
500

501
    global tempfolder
502
    currentfolder = os.getcwd()
503
    tempfolder = tempfile.mkdtemp()
504
    print("creating temp folder " + tempfolder)
505
    src = os.path.join(currentfolder, "freecad.zip")
506
    dst = os.path.join(tempfolder, "freecad.zip")
507
    if not os.path.exists(src):
508
        print('freecad.zip file not found! Aborting. Run "download" command before this one.')
509
        sys.exit()
510
    shutil.copyfile(src, dst)
511
    os.chdir(tempfolder)
512
    zfile = zipfile.ZipFile("freecad.zip")
513
    print("extracting freecad.zip...")
514
    zfile.extractall()
515
    os.chdir(currentfolder)
516
    for ln in languages:
517
        if not os.path.exists(os.path.join(tempfolder, ln)):
518
            print("ERROR: language path for " + ln + " not found!")
519
        else:
520
            doLanguage(ln)
521

522

523
if __name__ == "__main__":
524
    command = None
525

526
    args = sys.argv[1:]
527
    if args:
528
        command = args[0]
529

530
    token = os.environ.get("CROWDIN_TOKEN", load_token())
531
    if command and not token:
532
        print("Token not found")
533
        sys.exit()
534

535
    project_identifier = os.environ.get("CROWDIN_PROJECT_ID")
536
    if not project_identifier:
537
        project_identifier = "freecad"
538
        # print('CROWDIN_PROJECT_ID env var must be set')
539
        # sys.exit()
540

541
    updater = CrowdinUpdater(token, project_identifier)
542

543
    if command == "status":
544
        status = updater.status()
545
        status = sorted(status, key=lambda item: item["translationProgress"], reverse=True)
546
        print(
547
            len([item for item in status if item["translationProgress"] > THRESHOLD]),
548
            " languages with status > " + str(THRESHOLD) + "%:",
549
        )
550
        print("    ")
551
        sep = False
552
        prefix = ""
553
        suffix = ""
554
        if os.name == "posix":
555
            prefix = "\033[;32m"
556
            suffix = "\033[0m"
557
        for item in status:
558
            if item["translationProgress"] > 0:
559
                if (item["translationProgress"] < THRESHOLD) and (not sep):
560
                    print("    ")
561
                    print("Other languages:")
562
                    print("    ")
563
                    sep = True
564
                print(
565
                    prefix
566
                    + item["languageId"]
567
                    + suffix
568
                    + " "
569
                    + str(item["translationProgress"])
570
                    + "% ("
571
                    + str(item["approvalProgress"])
572
                    + "% approved)"
573
                )
574
                # print(f"  translation progress: {item['translationProgress']}%")
575
                # print(f"  approval progress:    {item['approvalProgress']}%")
576

577
    elif command == "build-status":
578
        for item in updater.build_status():
579
            print(f"  id: {item['id']} progress: {item['progress']}% status: {item['status']}")
580

581
    elif command == "build":
582
        updater.build()
583

584
    elif command == "download":
585
        if len(args) == 2:
586
            updater.download(args[1])
587
        else:
588
            stat = updater.build_status()
589
            if not stat:
590
                print("no builds found")
591
            elif len(stat) == 1:
592
                updater.download(stat[0]["id"])
593
            else:
594
                print("available builds:")
595
                for item in stat:
596
                    print(
597
                        f"  id: {item['id']} progress: {item['progress']}% status: {item['status']}"
598
                    )
599
                print("please specify a build id")
600

601
    elif command in ["update", "upload"]:
602
        # Find all ts files. However, this contains the lang-specific files too. Let's drop those
603
        all_ts_files = glob.glob("../**/*.ts", recursive=True)
604
        # Remove the file extensions
605
        ts_files_wo_ext = [splitext(f)[0] for f in all_ts_files]
606
        # Filter out any file that has another file as a substring. E.g. Draft is a substring of Draft_en
607
        main_ts_files = list(
608
            filter(
609
                lambda f: not [a for a in ts_files_wo_ext if a in f and f != a],
610
                ts_files_wo_ext,
611
            )
612
        )
613
        # Create tuples to map Crowdin name with local path name
614
        names_and_path = [(f"{basename(f)}.ts", f"{f}.ts") for f in main_ts_files]
615
        # Accommodate for legacy naming
616
        ts_files = [
617
            TsFile(LEGACY_NAMING_MAP[a] if a in LEGACY_NAMING_MAP else a, b)
618
            for (a, b) in names_and_path
619
        ]
620
        updater.update(ts_files)
621

622
    elif command in ["apply", "install"]:
623
        print("retrieving list of languages...")
624
        status = updater.status()
625
        status = sorted(status, key=lambda item: item["translationProgress"], reverse=True)
626
        languages = [
627
            item["languageId"] for item in status if item["translationProgress"] > THRESHOLD
628
        ]
629
        applyTranslations(languages)
630
        print("Updating Translator.cpp...")
631
        for ln in languages:
632
            updateTranslatorCpp(ln)
633

634
    elif command == "updateTranslator":
635
        print("retrieving list of languages...")
636
        status = updater.status()
637
        status = sorted(status, key=lambda item: item["translationProgress"], reverse=True)
638
        languages = [
639
            item["languageId"] for item in status if item["translationProgress"] > THRESHOLD
640
        ]
641
        print("Updating Translator.cpp...")
642
        for ln in languages:
643
            updateTranslatorCpp(ln)
644

645
    elif command == "gather":
646
        import updatets
647

648
        updatets.main()
649

650
    else:
651
        print(__doc__)
652