FreeCAD

1
# SPDX-License-Identifier: LGPL-2.1-or-later
2
# ***************************************************************************
3
# *                                                                         *
4
# *   Copyright (c) 2022-2023 FreeCAD Project Association                   *
5
# *   Copyright (c) 2019 Yorik van Havre <yorik@uncreated.net>              *
6
# *                                                                         *
7
# *   This file is part of FreeCAD.                                         *
8
# *                                                                         *
9
# *   FreeCAD is free software: you can redistribute it and/or modify it    *
10
# *   under the terms of the GNU Lesser General Public License as           *
11
# *   published by the Free Software Foundation, either version 2.1 of the  *
12
# *   License, or (at your option) any later version.                       *
13
# *                                                                         *
14
# *   FreeCAD is distributed in the hope that it will be useful, but        *
15
# *   WITHOUT ANY WARRANTY; without even the implied warranty of            *
16
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU      *
17
# *   Lesser General Public License for more details.                       *
18
# *                                                                         *
19
# *   You should have received a copy of the GNU Lesser General Public      *
20
# *   License along with FreeCAD. If not, see                               *
21
# *   <https://www.gnu.org/licenses/>.                                      *
22
# *                                                                         *
23
# ***************************************************************************
24

25
""" Worker thread classes for Addon Manager startup """
26
import hashlib
27
import json
28
import os
29
import queue
30
import re
31
import shutil
32
import stat
33
import threading
34
import time
35
from typing import List
36

37
from PySide import QtCore
38

39
import FreeCAD
40
import addonmanager_utilities as utils
41
from addonmanager_macro import Macro
42
from Addon import Addon
43
from AddonStats import AddonStats
44
import NetworkManager
45
from addonmanager_git import initialize_git, GitFailed
46
from addonmanager_metadata import MetadataReader, get_branch_from_metadata
47

48
translate = FreeCAD.Qt.translate
49

50
# Workers only have one public method by design
51
# pylint: disable=c-extension-no-member,too-few-public-methods,too-many-instance-attributes
52

53

54
class CreateAddonListWorker(QtCore.QThread):
55
    """This worker updates the list of available workbenches, emitting an "addon_repo"
56
    signal for each Addon as they are processed."""
57

58
    status_message = QtCore.Signal(str)
59
    addon_repo = QtCore.Signal(object)
60

61
    def __init__(self):
62
        QtCore.QThread.__init__(self)
63

64
        # reject_listed addons
65
        self.macros_reject_list = []
66
        self.mod_reject_list = []
67

68
        # These addons will print an additional message informing the user
69
        self.obsolete = []
70

71
        # These addons will print an additional message informing the user Python2 only
72
        self.py2only = []
73

74
        self.package_names = []
75
        self.moddir = os.path.join(FreeCAD.getUserAppDataDir(), "Mod")
76
        self.current_thread = None
77

78
        self.git_manager = initialize_git()
79

80
    def run(self):
81
        "populates the list of addons"
82

83
        self.current_thread = QtCore.QThread.currentThread()
84
        try:
85
            self._get_freecad_addon_repo_data()
86
        except ConnectionError:
87
            return
88
        self._get_custom_addons()
89
        self._get_official_addons()
90
        self._retrieve_macros_from_git()
91
        self._retrieve_macros_from_wiki()
92

93
    def _get_freecad_addon_repo_data(self):
94
        # update info lists
95
        p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(
96
            "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json", 5000
97
        )
98
        if p:
99
            p = p.data().decode("utf8")
100
            j = json.loads(p)
101
            if "obsolete" in j and "Mod" in j["obsolete"]:
102
                self.obsolete = j["obsolete"]["Mod"]
103

104
            if "blacklisted" in j and "Macro" in j["blacklisted"]:
105
                self.macros_reject_list = j["blacklisted"]["Macro"]
106

107
            if "blacklisted" in j and "Mod" in j["blacklisted"]:
108
                self.mod_reject_list = j["blacklisted"]["Mod"]
109

110
            if "py2only" in j and "Mod" in j["py2only"]:
111
                self.py2only = j["py2only"]["Mod"]
112

113
            if "deprecated" in j:
114
                self._process_deprecated(j["deprecated"])
115

116
        else:
117
            message = translate(
118
                "AddonsInstaller",
119
                "Failed to connect to GitHub. Check your connection and proxy settings.",
120
            )
121
            FreeCAD.Console.PrintError(message + "\n")
122
            self.status_message.emit(message)
123
            raise ConnectionError
124

125
    def _process_deprecated(self, deprecated_addons):
126
        """Parse the section on deprecated addons"""
127

128
        fc_major = int(FreeCAD.Version()[0])
129
        fc_minor = int(FreeCAD.Version()[1])
130
        for item in deprecated_addons:
131
            if "as_of" in item and "name" in item:
132
                try:
133
                    version_components = item["as_of"].split(".")
134
                    major = int(version_components[0])
135
                    if len(version_components) > 1:
136
                        minor = int(version_components[1])
137
                    else:
138
                        minor = 0
139
                    if major < fc_major or (major == fc_major and minor <= fc_minor):
140
                        if "kind" not in item or item["kind"] == "mod":
141
                            self.obsolete.append(item["name"])
142
                        elif item["kind"] == "macro":
143
                            self.macros_reject_list.append(item["name"])
144
                        else:
145
                            FreeCAD.Console.PrintMessage(
146
                                f'Unrecognized Addon kind {item["kind"]} in deprecation list.'
147
                            )
148
                except ValueError:
149
                    FreeCAD.Console.PrintMessage(
150
                        f"Failed to parse version from {item['name']}, version {item['as_of']}"
151
                    )
152

153
    def _get_custom_addons(self):
154

155
        # querying custom addons first
156
        addon_list = (
157
            FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
158
            .GetString("CustomRepositories", "")
159
            .split("\n")
160
        )
161
        custom_addons = []
162
        for addon in addon_list:
163
            if " " in addon:
164
                addon_and_branch = addon.split(" ")
165
                custom_addons.append({"url": addon_and_branch[0], "branch": addon_and_branch[1]})
166
            else:
167
                custom_addons.append({"url": addon, "branch": "master"})
168
        for addon in custom_addons:
169
            if self.current_thread.isInterruptionRequested():
170
                return
171
            if addon and addon["url"]:
172
                if addon["url"][-1] == "/":
173
                    addon["url"] = addon["url"][0:-1]  # Strip trailing slash
174
                addon["url"] = addon["url"].split(".git")[0]  # Remove .git
175
                name = addon["url"].split("/")[-1]
176
                if name in self.package_names:
177
                    # We already have something with this name, skip this one
178
                    FreeCAD.Console.PrintWarning(
179
                        translate("AddonsInstaller", "WARNING: Duplicate addon {} ignored").format(
180
                            name
181
                        )
182
                    )
183
                    continue
184
                FreeCAD.Console.PrintLog(
185
                    f"Adding custom location {addon['url']} with branch {addon['branch']}\n"
186
                )
187
                self.package_names.append(name)
188
                addondir = os.path.join(self.moddir, name)
189
                if os.path.exists(addondir) and os.listdir(addondir):
190
                    state = Addon.Status.UNCHECKED
191
                else:
192
                    state = Addon.Status.NOT_INSTALLED
193
                repo = Addon(name, addon["url"], state, addon["branch"])
194
                md_file = os.path.join(addondir, "package.xml")
195
                if os.path.isfile(md_file):
196
                    repo.installed_metadata = MetadataReader.from_file(md_file)
197
                    repo.installed_version = repo.installed_metadata.version
198
                    repo.updated_timestamp = os.path.getmtime(md_file)
199
                    repo.verify_url_and_branch(addon["url"], addon["branch"])
200

201
                self.addon_repo.emit(repo)
202

203
    def _get_official_addons(self):
204
        # querying official addons
205
        p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(
206
            "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules", 5000
207
        )
208
        if not p:
209
            return
210
        p = p.data().decode("utf8")
211
        p = re.findall(
212
            (
213
                r'(?m)\[submodule\s*"(?P<name>.*)"\]\s*'
214
                r"path\s*=\s*(?P<path>.+)\s*"
215
                r"url\s*=\s*(?P<url>https?://.*)\s*"
216
                r"(branch\s*=\s*(?P<branch>[^\s]*)\s*)?"
217
            ),
218
            p,
219
        )
220
        for name, _, url, _, branch in p:
221
            if self.current_thread.isInterruptionRequested():
222
                return
223
            if name in self.package_names:
224
                # We already have something with this name, skip this one
225
                continue
226
            self.package_names.append(name)
227
            if branch is None or len(branch) == 0:
228
                branch = "master"
229
            url = url.split(".git")[0]
230
            addondir = os.path.join(self.moddir, name)
231
            if os.path.exists(addondir) and os.listdir(addondir):
232
                # make sure the folder exists and it contains files!
233
                state = Addon.Status.UNCHECKED
234
            else:
235
                state = Addon.Status.NOT_INSTALLED
236
            repo = Addon(name, url, state, branch)
237
            md_file = os.path.join(addondir, "package.xml")
238
            if os.path.isfile(md_file):
239
                repo.installed_metadata = MetadataReader.from_file(md_file)
240
                repo.installed_version = repo.installed_metadata.version
241
                repo.updated_timestamp = os.path.getmtime(md_file)
242
                repo.verify_url_and_branch(url, branch)
243

244
            if name in self.py2only:
245
                repo.python2 = True
246
            if name in self.mod_reject_list:
247
                repo.rejected = True
248
            if name in self.obsolete:
249
                repo.obsolete = True
250
            self.addon_repo.emit(repo)
251

252
            self.status_message.emit(translate("AddonsInstaller", "Workbenches list was updated."))
253

254
    def _retrieve_macros_from_git(self):
255
        """Retrieve macros from FreeCAD-macros.git
256

257
        Emits a signal for each macro in
258
        https://github.com/FreeCAD/FreeCAD-macros.git
259
        """
260

261
        macro_cache_location = utils.get_cache_file_name("Macros")
262

263
        if not self.git_manager:
264
            message = translate(
265
                "AddonsInstaller",
266
                "Git is disabled, skipping git macros",
267
            )
268
            self.status_message.emit(message)
269
            FreeCAD.Console.PrintWarning(message + "\n")
270
            return
271

272
        update_succeeded = self._update_local_git_repo()
273
        if not update_succeeded:
274
            return
275

276
        n_files = 0
277
        for _, _, filenames in os.walk(macro_cache_location):
278
            n_files += len(filenames)
279
        counter = 0
280
        for dirpath, _, filenames in os.walk(macro_cache_location):
281
            counter += 1
282
            if self.current_thread.isInterruptionRequested():
283
                return
284
            if ".git" in dirpath:
285
                continue
286
            for filename in filenames:
287
                if self.current_thread.isInterruptionRequested():
288
                    return
289
                if filename.lower().endswith(".fcmacro"):
290
                    macro = Macro(filename[:-8])  # Remove ".FCMacro".
291
                    if macro.name in self.package_names:
292
                        FreeCAD.Console.PrintLog(
293
                            f"Ignoring second macro named {macro.name} (found on git)\n"
294
                        )
295
                        continue  # We already have a macro with this name
296
                    self.package_names.append(macro.name)
297
                    macro.on_git = True
298
                    macro.src_filename = os.path.join(dirpath, filename)
299
                    macro.fill_details_from_file(macro.src_filename)
300
                    repo = Addon.from_macro(macro)
301
                    FreeCAD.Console.PrintLog(f"Found macro {repo.name}\n")
302
                    repo.url = "https://github.com/FreeCAD/FreeCAD-macros.git"
303
                    utils.update_macro_installation_details(repo)
304
                    self.addon_repo.emit(repo)
305

306
    def _update_local_git_repo(self) -> bool:
307
        macro_cache_location = utils.get_cache_file_name("Macros")
308
        try:
309
            if os.path.exists(macro_cache_location):
310
                if not os.path.exists(os.path.join(macro_cache_location, ".git")):
311
                    FreeCAD.Console.PrintWarning(
312
                        translate(
313
                            "AddonsInstaller",
314
                            "Attempting to change non-git Macro setup to use git\n",
315
                        )
316
                    )
317
                    self.git_manager.repair(
318
                        "https://github.com/FreeCAD/FreeCAD-macros.git",
319
                        macro_cache_location,
320
                    )
321
                self.git_manager.update(macro_cache_location)
322
            else:
323
                self.git_manager.clone(
324
                    "https://github.com/FreeCAD/FreeCAD-macros.git",
325
                    macro_cache_location,
326
                )
327
        except GitFailed as e:
328
            FreeCAD.Console.PrintMessage(
329
                translate(
330
                    "AddonsInstaller",
331
                    "An error occurred updating macros from GitHub, trying clean checkout...",
332
                )
333
                + f":\n{e}\n"
334
            )
335
            FreeCAD.Console.PrintMessage(f"{macro_cache_location}\n")
336
            FreeCAD.Console.PrintMessage(
337
                translate("AddonsInstaller", "Attempting to do a clean checkout...") + "\n"
338
            )
339
            try:
340
                os.chdir(
341
                    os.path.join(macro_cache_location, "..")
342
                )  # Make sure we are not IN this directory
343
                shutil.rmtree(macro_cache_location, onerror=self._remove_readonly)
344
                self.git_manager.clone(
345
                    "https://github.com/FreeCAD/FreeCAD-macros.git",
346
                    macro_cache_location,
347
                )
348
                FreeCAD.Console.PrintMessage(
349
                    translate("AddonsInstaller", "Clean checkout succeeded") + "\n"
350
                )
351
            except GitFailed as e2:
352
                # The Qt Python translation extractor doesn't support splitting this string (yet)
353
                # pylint: disable=line-too-long
354
                FreeCAD.Console.PrintWarning(
355
                    translate(
356
                        "AddonsInstaller",
357
                        "Failed to update macros from GitHub -- try clearing the Addon Manager's cache.",
358
                    )
359
                    + f":\n{str(e2)}\n"
360
                )
361
                return False
362
        return True
363

364
    def _retrieve_macros_from_wiki(self):
365
        """Retrieve macros from the wiki
366

367
        Read the wiki and emit a signal for each found macro.
368
        Reads only the page https://wiki.freecad.org/Macros_recipes
369
        """
370

371
        p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(
372
            "https://wiki.freecad.org/Macros_recipes", 5000
373
        )
374
        if not p:
375
            # The Qt Python translation extractor doesn't support splitting this string (yet)
376
            # pylint: disable=line-too-long
377
            FreeCAD.Console.PrintWarning(
378
                translate(
379
                    "AddonsInstaller",
380
                    "Error connecting to the Wiki, FreeCAD cannot retrieve the Wiki macro list at this time",
381
                )
382
                + "\n"
383
            )
384
            return
385
        p = p.data().decode("utf8")
386
        macros = re.findall('title="(Macro.*?)"', p)
387
        macros = [mac for mac in macros if "translated" not in mac]
388
        macro_names = []
389
        for _, mac in enumerate(macros):
390
            if self.current_thread.isInterruptionRequested():
391
                return
392
            macname = mac[6:]  # Remove "Macro ".
393
            macname = macname.replace("&amp;", "&")
394
            if not macname:
395
                continue
396
            if (
397
                (macname not in self.macros_reject_list)
398
                and ("recipes" not in macname.lower())
399
                and (macname not in macro_names)
400
            ):
401
                macro_names.append(macname)
402
                macro = Macro(macname)
403
                if macro.name in self.package_names:
404
                    FreeCAD.Console.PrintLog(
405
                        f"Ignoring second macro named {macro.name} (found on wiki)\n"
406
                    )
407
                    continue  # We already have a macro with this name
408
                self.package_names.append(macro.name)
409
                macro.on_wiki = True
410
                macro.parsed = False
411
                repo = Addon.from_macro(macro)
412
                repo.url = "https://wiki.freecad.org/Macros_recipes"
413
                utils.update_macro_installation_details(repo)
414
                self.addon_repo.emit(repo)
415

416
    def _remove_readonly(self, func, path, _) -> None:
417
        """Remove a read-only file."""
418

419
        os.chmod(path, stat.S_IWRITE)
420
        func(path)
421

422

423
class LoadPackagesFromCacheWorker(QtCore.QThread):
424
    """A subthread worker that loads package information from its cache file."""
425

426
    addon_repo = QtCore.Signal(object)
427

428
    def __init__(self, cache_file: str):
429
        QtCore.QThread.__init__(self)
430
        self.cache_file = cache_file
431
        self.metadata_cache_path = os.path.join(
432
            FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata"
433
        )
434

435
    def override_metadata_cache_path(self, path):
436
        """For testing purposes, override the location to fetch the package metadata from."""
437
        self.metadata_cache_path = path
438

439
    def run(self):
440
        """Rarely called directly: create an instance and call start() on it instead to
441
        launch in a new thread"""
442
        with open(self.cache_file, encoding="utf-8") as f:
443
            data = f.read()
444
            if data:
445
                dict_data = json.loads(data)
446
                for item in dict_data.values():
447
                    if QtCore.QThread.currentThread().isInterruptionRequested():
448
                        return
449
                    repo = Addon.from_cache(item)
450
                    repo_metadata_cache_path = os.path.join(
451
                        self.metadata_cache_path, repo.name, "package.xml"
452
                    )
453
                    if os.path.isfile(repo_metadata_cache_path):
454
                        try:
455
                            repo.load_metadata_file(repo_metadata_cache_path)
456
                        except RuntimeError as e:
457
                            FreeCAD.Console.PrintLog(f"Failed loading {repo_metadata_cache_path}\n")
458
                            FreeCAD.Console.PrintLog(str(e) + "\n")
459
                    self.addon_repo.emit(repo)
460

461

462
class LoadMacrosFromCacheWorker(QtCore.QThread):
463
    """A worker object to load macros from a cache file"""
464

465
    add_macro_signal = QtCore.Signal(object)
466

467
    def __init__(self, cache_file: str):
468
        QtCore.QThread.__init__(self)
469
        self.cache_file = cache_file
470

471
    def run(self):
472
        """Rarely called directly: create an instance and call start() on it instead to
473
        launch in a new thread"""
474

475
        with open(self.cache_file, encoding="utf-8") as f:
476
            data = f.read()
477
            dict_data = json.loads(data)
478
            for item in dict_data:
479
                if QtCore.QThread.currentThread().isInterruptionRequested():
480
                    return
481
                new_macro = Macro.from_cache(item)
482
                repo = Addon.from_macro(new_macro)
483
                utils.update_macro_installation_details(repo)
484
                self.add_macro_signal.emit(repo)
485

486

487
class CheckSingleUpdateWorker(QtCore.QObject):
488
    """This worker is a little different from the others: the actual recommended way of
489
    running in a QThread is to make a worker object that gets moved into the thread."""
490

491
    update_status = QtCore.Signal(int)
492

493
    def __init__(self, repo: Addon, parent: QtCore.QObject = None):
494
        super().__init__(parent)
495
        self.repo = repo
496

497
    def do_work(self):
498
        """Use the UpdateChecker class to do the work of this function, depending on the
499
        type of Addon"""
500

501
        checker = UpdateChecker()
502
        if self.repo.repo_type == Addon.Kind.WORKBENCH:
503
            checker.check_workbench(self.repo)
504
        elif self.repo.repo_type == Addon.Kind.MACRO:
505
            checker.check_macro(self.repo)
506
        elif self.repo.repo_type == Addon.Kind.PACKAGE:
507
            checker.check_package(self.repo)
508

509
        self.update_status.emit(self.repo.update_status)
510

511

512
class CheckWorkbenchesForUpdatesWorker(QtCore.QThread):
513
    """This worker checks for available updates for all workbenches"""
514

515
    update_status = QtCore.Signal(Addon)
516
    progress_made = QtCore.Signal(int, int)
517

518
    def __init__(self, repos: List[Addon]):
519

520
        QtCore.QThread.__init__(self)
521
        self.repos = repos
522
        self.current_thread = None
523
        self.basedir = FreeCAD.getUserAppDataDir()
524
        self.moddir = os.path.join(self.basedir, "Mod")
525

526
    def run(self):
527
        """Rarely called directly: create an instance and call start() on it instead to
528
        launch in a new thread"""
529

530
        self.current_thread = QtCore.QThread.currentThread()
531
        checker = UpdateChecker()
532
        count = 1
533
        for repo in self.repos:
534
            if self.current_thread.isInterruptionRequested():
535
                return
536
            self.progress_made.emit(count, len(self.repos))
537
            count += 1
538
            if repo.status() == Addon.Status.UNCHECKED:
539
                if repo.repo_type == Addon.Kind.WORKBENCH:
540
                    checker.check_workbench(repo)
541
                    self.update_status.emit(repo)
542
                elif repo.repo_type == Addon.Kind.MACRO:
543
                    checker.check_macro(repo)
544
                    self.update_status.emit(repo)
545
                elif repo.repo_type == Addon.Kind.PACKAGE:
546
                    checker.check_package(repo)
547
                    self.update_status.emit(repo)
548

549

550
class UpdateChecker:
551
    """A utility class used by the CheckWorkbenchesForUpdatesWorker class. Each function is
552
    designed for a specific Addon type, and modifies the passed-in Addon with the determined
553
    update status."""
554

555
    def __init__(self):
556
        self.basedir = FreeCAD.getUserAppDataDir()
557
        self.moddir = os.path.join(self.basedir, "Mod")
558
        self.git_manager = initialize_git()
559

560
    def override_mod_directory(self, moddir):
561
        """Primarily for use when testing, sets an alternate directory to use for mods"""
562
        self.moddir = moddir
563

564
    def check_workbench(self, wb):
565
        """Given a workbench Addon wb, check it for updates using git. If git is not
566
        available, does nothing."""
567
        if not self.git_manager:
568
            wb.set_status(Addon.Status.CANNOT_CHECK)
569
            return
570
        clonedir = os.path.join(self.moddir, wb.name)
571
        if os.path.exists(clonedir):
572
            # mark as already installed AND already checked for updates
573
            if not os.path.exists(os.path.join(clonedir, ".git")):
574
                with wb.git_lock:
575
                    self.git_manager.repair(wb.url, clonedir)
576
            with wb.git_lock:
577
                try:
578
                    status = self.git_manager.status(clonedir)
579
                    if "(no branch)" in status:
580
                        # By definition, in a detached-head state we cannot
581
                        # update, so don't even bother checking.
582
                        wb.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
583
                        wb.branch = self.git_manager.current_branch(clonedir)
584
                        return
585
                except GitFailed as e:
586
                    FreeCAD.Console.PrintWarning(
587
                        "AddonManager: "
588
                        + translate(
589
                            "AddonsInstaller",
590
                            "Unable to fetch git updates for workbench {}",
591
                        ).format(wb.name)
592
                        + "\n"
593
                    )
594
                    FreeCAD.Console.PrintWarning(str(e) + "\n")
595
                    wb.set_status(Addon.Status.CANNOT_CHECK)
596
                else:
597
                    try:
598
                        if self.git_manager.update_available(clonedir):
599
                            wb.set_status(Addon.Status.UPDATE_AVAILABLE)
600
                        else:
601
                            wb.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
602
                    except GitFailed:
603
                        FreeCAD.Console.PrintWarning(
604
                            translate("AddonsInstaller", "git status failed for {}").format(wb.name)
605
                            + "\n"
606
                        )
607
                        wb.set_status(Addon.Status.CANNOT_CHECK)
608

609
    def _branch_name_changed(self, package: Addon) -> bool:
610
        clone_dir = os.path.join(self.moddir, package.name)
611
        installed_metadata_file = os.path.join(clone_dir, "package.xml")
612
        if not os.path.isfile(installed_metadata_file):
613
            return False
614
        if not hasattr(package, "metadata") or package.metadata is None:
615
            return False
616
        try:
617
            installed_metadata = MetadataReader.from_file(installed_metadata_file)
618
            installed_default_branch = get_branch_from_metadata(installed_metadata)
619
            remote_default_branch = get_branch_from_metadata(package.metadata)
620
            if installed_default_branch != remote_default_branch:
621
                return True
622
        except RuntimeError:
623
            return False
624
        return False
625

626
    def check_package(self, package: Addon) -> None:
627
        """Given a packaged Addon package, check it for updates. If git is available that is
628
        used. If not, the package's metadata is examined, and if the metadata file has changed
629
        compared to the installed copy, an update is flagged. In addition, a change to the
630
        default branch name triggers an update."""
631

632
        clone_dir = self.moddir + os.sep + package.name
633
        if os.path.exists(clone_dir):
634

635
            # First, see if the branch name changed, which automatically triggers an update
636
            if self._branch_name_changed(package):
637
                package.set_status(Addon.Status.UPDATE_AVAILABLE)
638
                return
639

640
            # Next, try to just do a git-based update, which will give the most accurate results:
641
            if self.git_manager:
642
                self.check_workbench(package)
643
                if package.status() != Addon.Status.CANNOT_CHECK:
644
                    # It worked, just exit now
645
                    return
646

647
            # If we were unable to do a git-based update, try using the package.xml file instead:
648
            installed_metadata_file = os.path.join(clone_dir, "package.xml")
649
            if not os.path.isfile(installed_metadata_file):
650
                # If there is no package.xml file, then it's because the package author added it
651
                # after the last time the local installation was updated. By definition, then,
652
                # there is an update available, if only to download the new XML file.
653
                package.set_status(Addon.Status.UPDATE_AVAILABLE)
654
                package.installed_version = None
655
                return
656
            package.updated_timestamp = os.path.getmtime(installed_metadata_file)
657
            try:
658
                installed_metadata = MetadataReader.from_file(installed_metadata_file)
659
                package.installed_version = installed_metadata.version
660
                # Packages are considered up-to-date if the metadata version matches.
661
                # Authors should update their version string when they want the addon
662
                # manager to alert users of a new version.
663
                if package.metadata.version != installed_metadata.version:
664
                    package.set_status(Addon.Status.UPDATE_AVAILABLE)
665
                else:
666
                    package.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
667
            except RuntimeError:
668
                FreeCAD.Console.PrintWarning(
669
                    translate(
670
                        "AddonsInstaller",
671
                        "Failed to read metadata from {name}",
672
                    ).format(name=installed_metadata_file)
673
                    + "\n"
674
                )
675
                package.set_status(Addon.Status.CANNOT_CHECK)
676

677
    def check_macro(self, macro_wrapper: Addon) -> None:
678
        """Check to see if the online copy of the macro's code differs from the local copy."""
679

680
        # Make sure this macro has its code downloaded:
681
        try:
682
            if not macro_wrapper.macro.parsed and macro_wrapper.macro.on_git:
683
                macro_wrapper.macro.fill_details_from_file(macro_wrapper.macro.src_filename)
684
            elif not macro_wrapper.macro.parsed and macro_wrapper.macro.on_wiki:
685
                mac = macro_wrapper.macro.name.replace(" ", "_")
686
                mac = mac.replace("&", "%26")
687
                mac = mac.replace("+", "%2B")
688
                url = "https://wiki.freecad.org/Macro_" + mac
689
                macro_wrapper.macro.fill_details_from_wiki(url)
690
        except RuntimeError:
691
            FreeCAD.Console.PrintWarning(
692
                translate(
693
                    "AddonsInstaller",
694
                    "Failed to fetch code for macro '{name}'",
695
                ).format(name=macro_wrapper.macro.name)
696
                + "\n"
697
            )
698
            macro_wrapper.set_status(Addon.Status.CANNOT_CHECK)
699
            return
700

701
        hasher1 = hashlib.sha1()
702
        hasher2 = hashlib.sha1()
703
        hasher1.update(macro_wrapper.macro.code.encode("utf-8"))
704
        new_sha1 = hasher1.hexdigest()
705
        test_file_one = os.path.join(FreeCAD.getUserMacroDir(True), macro_wrapper.macro.filename)
706
        test_file_two = os.path.join(
707
            FreeCAD.getUserMacroDir(True), "Macro_" + macro_wrapper.macro.filename
708
        )
709
        if os.path.exists(test_file_one):
710
            with open(test_file_one, "rb") as f:
711
                contents = f.read()
712
                hasher2.update(contents)
713
                old_sha1 = hasher2.hexdigest()
714
        elif os.path.exists(test_file_two):
715
            with open(test_file_two, "rb") as f:
716
                contents = f.read()
717
                hasher2.update(contents)
718
                old_sha1 = hasher2.hexdigest()
719
        else:
720
            return
721
        if new_sha1 == old_sha1:
722
            macro_wrapper.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
723
        else:
724
            macro_wrapper.set_status(Addon.Status.UPDATE_AVAILABLE)
725

726

727
class CacheMacroCodeWorker(QtCore.QThread):
728
    """Download and cache the macro code, and parse its internal metadata"""
729

730
    status_message = QtCore.Signal(str)
731
    update_macro = QtCore.Signal(Addon)
732
    progress_made = QtCore.Signal(int, int)
733

734
    def __init__(self, repos: List[Addon]) -> None:
735
        QtCore.QThread.__init__(self)
736
        self.repos = repos
737
        self.workers = []
738
        self.terminators = []
739
        self.lock = threading.Lock()
740
        self.failed = []
741
        self.counter = 0
742
        self.repo_queue = None
743

744
    def run(self):
745
        """Rarely called directly: create an instance and call start() on it instead to
746
        launch in a new thread"""
747

748
        self.status_message.emit(translate("AddonsInstaller", "Caching macro code..."))
749

750
        self.repo_queue = queue.Queue()
751
        num_macros = 0
752
        for repo in self.repos:
753
            if repo.macro is not None:
754
                self.repo_queue.put(repo)
755
                num_macros += 1
756

757
        interrupted = self._process_queue(num_macros)
758
        if interrupted:
759
            return
760

761
        # Make sure all of our child threads have fully exited:
762
        for worker in self.workers:
763
            worker.wait(50)
764
            if not worker.isFinished():
765
                # The Qt Python translation extractor doesn't support splitting this string (yet)
766
                # pylint: disable=line-too-long
767
                FreeCAD.Console.PrintError(
768
                    translate(
769
                        "AddonsInstaller",
770
                        "Addon Manager: a worker process failed to complete while fetching {name}",
771
                    ).format(name=worker.macro.name)
772
                    + "\n"
773
                )
774

775
        self.repo_queue.join()
776
        for terminator in self.terminators:
777
            if terminator and terminator.isActive():
778
                terminator.stop()
779

780
        if len(self.failed) > 0:
781
            num_failed = len(self.failed)
782
            FreeCAD.Console.PrintWarning(
783
                translate(
784
                    "AddonsInstaller",
785
                    "Out of {num_macros} macros, {num_failed} timed out while processing",
786
                ).format(num_macros=num_macros, num_failed=num_failed)
787
                + "\n"
788
            )
789

790
    def _process_queue(self, num_macros) -> bool:
791
        """Spools up six network connections and downloads the macro code. Returns True if
792
        it was interrupted by user request, or False if it ran to completion."""
793

794
        # Emulate QNetworkAccessManager and spool up six connections:
795
        for _ in range(6):
796
            self.update_and_advance(None)
797

798
        current_thread = QtCore.QThread.currentThread()
799
        while True:
800
            if current_thread.isInterruptionRequested():
801
                for worker in self.workers:
802
                    worker.blockSignals(True)
803
                    worker.requestInterruption()
804
                    if not worker.wait(100):
805
                        FreeCAD.Console.PrintWarning(
806
                            translate(
807
                                "AddonsInstaller",
808
                                "Addon Manager: a worker process failed to halt ({name})",
809
                            ).format(name=worker.macro.name)
810
                            + "\n"
811
                        )
812
                return True
813
            # Ensure our signals propagate out by running an internal thread-local event loop
814
            QtCore.QCoreApplication.processEvents()
815
            with self.lock:
816
                if self.counter >= num_macros:
817
                    break
818
            time.sleep(0.1)
819
        return False
820

821
    def update_and_advance(self, repo: Addon) -> None:
822
        """Emit the updated signal and launch the next item from the queue."""
823
        if repo is not None:
824
            if repo.macro.name not in self.failed:
825
                self.update_macro.emit(repo)
826
            self.repo_queue.task_done()
827
            with self.lock:
828
                self.counter += 1
829

830
        if QtCore.QThread.currentThread().isInterruptionRequested():
831
            return
832

833
        self.progress_made.emit(len(self.repos) - self.repo_queue.qsize(), len(self.repos))
834

835
        try:
836
            next_repo = self.repo_queue.get_nowait()
837
            worker = GetMacroDetailsWorker(next_repo)
838
            worker.finished.connect(lambda: self.update_and_advance(next_repo))
839
            with self.lock:
840
                self.workers.append(worker)
841
                self.terminators.append(
842
                    QtCore.QTimer.singleShot(10000, lambda: self.terminate(worker))
843
                )
844
            self.status_message.emit(
845
                translate(
846
                    "AddonsInstaller",
847
                    "Getting metadata from macro {}",
848
                ).format(next_repo.macro.name)
849
            )
850
            worker.start()
851
        except queue.Empty:
852
            pass
853

854
    def terminate(self, worker) -> None:
855
        """Shut down all running workers and exit the thread"""
856
        if not worker.isFinished():
857
            macro_name = worker.macro.name
858
            FreeCAD.Console.PrintWarning(
859
                translate(
860
                    "AddonsInstaller",
861
                    "Timeout while fetching metadata for macro {}",
862
                ).format(macro_name)
863
                + "\n"
864
            )
865
            # worker.blockSignals(True)
866
            worker.requestInterruption()
867
            worker.wait(100)
868
            if worker.isRunning():
869
                FreeCAD.Console.PrintError(
870
                    translate(
871
                        "AddonsInstaller",
872
                        "Failed to kill process for macro {}!\n",
873
                    ).format(macro_name)
874
                )
875
            with self.lock:
876
                self.failed.append(macro_name)
877

878

879
class GetMacroDetailsWorker(QtCore.QThread):
880
    """Retrieve the macro details for a macro"""
881

882
    status_message = QtCore.Signal(str)
883
    readme_updated = QtCore.Signal(str)
884

885
    def __init__(self, repo):
886

887
        QtCore.QThread.__init__(self)
888
        self.macro = repo.macro
889

890
    def run(self):
891
        """Rarely called directly: create an instance and call start() on it instead to
892
        launch in a new thread"""
893

894
        self.status_message.emit(translate("AddonsInstaller", "Retrieving macro description..."))
895
        if not self.macro.parsed and self.macro.on_git:
896
            self.status_message.emit(translate("AddonsInstaller", "Retrieving info from git"))
897
            self.macro.fill_details_from_file(self.macro.src_filename)
898
        if not self.macro.parsed and self.macro.on_wiki:
899
            self.status_message.emit(translate("AddonsInstaller", "Retrieving info from wiki"))
900
            mac = self.macro.name.replace(" ", "_")
901
            mac = mac.replace("&", "%26")
902
            mac = mac.replace("+", "%2B")
903
            url = "https://wiki.freecad.org/Macro_" + mac
904
            self.macro.fill_details_from_wiki(url)
905
        message = (
906
            "<h1>"
907
            + self.macro.name
908
            + "</h1>"
909
            + self.macro.desc
910
            + '<br/><br/>Macro location: <a href="'
911
            + self.macro.url
912
            + '">'
913
            + self.macro.url
914
            + "</a>"
915
        )
916
        if QtCore.QThread.currentThread().isInterruptionRequested():
917
            return
918
        self.readme_updated.emit(message)
919

920

921
class GetBasicAddonStatsWorker(QtCore.QThread):
922
    """Fetch data from an addon stats repository."""
923

924
    update_addon_stats = QtCore.Signal(Addon)
925

926
    def __init__(self, url: str, addons: List[Addon], parent: QtCore.QObject = None):
927
        super().__init__(parent)
928
        self.url = url
929
        self.addons = addons
930

931
    def run(self):
932
        """Fetch the remote data and load it into the addons"""
933

934
        fetch_result = NetworkManager.AM_NETWORK_MANAGER.blocking_get(self.url, 5000)
935
        if fetch_result is None:
936
            FreeCAD.Console.PrintError(
937
                translate(
938
                    "AddonsInstaller",
939
                    "Failed to get Addon statistics from {} -- only sorting alphabetically will"
940
                    " be accurate\n",
941
                ).format(self.url)
942
            )
943
            return
944
        text_result = fetch_result.data().decode("utf8")
945
        json_result = json.loads(text_result)
946

947
        for addon in self.addons:
948
            if addon.url in json_result:
949
                addon.stats = AddonStats.from_json(json_result[addon.url])
950
                self.update_addon_stats.emit(addon)
951

952

953
class GetAddonScoreWorker(QtCore.QThread):
954
    """Fetch data from an addon score file."""
955

956
    update_addon_score = QtCore.Signal(Addon)
957

958
    def __init__(self, url: str, addons: List[Addon], parent: QtCore.QObject = None):
959
        super().__init__(parent)
960
        self.url = url
961
        self.addons = addons
962

963
    def run(self):
964
        """Fetch the remote data and load it into the addons"""
965

966
        if self.url != "TEST":
967
            fetch_result = NetworkManager.AM_NETWORK_MANAGER.blocking_get(self.url, 5000)
968
            if fetch_result is None:
969
                FreeCAD.Console.PrintError(
970
                    translate(
971
                        "AddonsInstaller",
972
                        "Failed to get Addon score from '{}' -- sorting by score will fail\n",
973
                    ).format(self.url)
974
                )
975
                return
976
            text_result = fetch_result.data().decode("utf8")
977
            json_result = json.loads(text_result)
978
        else:
979
            FreeCAD.Console.PrintWarning("Running score generation in TEST mode...\n")
980
            json_result = {}
981
            for addon in self.addons:
982
                if addon.macro:
983
                    json_result[addon.name] = len(addon.macro.comment) if addon.macro.comment else 0
984
                else:
985
                    json_result[addon.url] = len(addon.description) if addon.description else 0
986

987
        for addon in self.addons:
988
            score = None
989
            if addon.url in json_result:
990
                score = json_result[addon.url]
991
            elif addon.name in json_result:
992
                score = json_result[addon.name]
993
            if score is not None:
994
                try:
995
                    addon.score = int(score)
996
                    self.update_addon_score.emit(addon)
997
                except (ValueError, OverflowError):
998
                    FreeCAD.Console.PrintLog(
999
                        f"Failed to convert score value '{score}' to an integer for {addon.name}"
1000
                    )
1001