FreeCAD

1
# SPDX-License-Identifier: LGPL-2.1-or-later
2
# ***************************************************************************
3
# *                                                                         *
4
# *   Copyright (c) 2022-2023 FreeCAD Project Association                   *
5
# *   Copyright (c) 2019 Yorik van Havre <yorik@uncreated.net>              *
6
# *                                                                         *
7
# *   This file is part of FreeCAD.                                         *
8
# *                                                                         *
9
# *   FreeCAD is free software: you can redistribute it and/or modify it    *
10
# *   under the terms of the GNU Lesser General Public License as           *
11
# *   published by the Free Software Foundation, either version 2.1 of the  *
12
# *   License, or (at your option) any later version.                       *
13
# *                                                                         *
14
# *   FreeCAD is distributed in the hope that it will be useful, but        *
15
# *   WITHOUT ANY WARRANTY; without even the implied warranty of            *
16
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU      *
17
# *   Lesser General Public License for more details.                       *
18
# *                                                                         *
19
# *   You should have received a copy of the GNU Lesser General Public      *
20
# *   License along with FreeCAD. If not, see                               *
21
# *   <https://www.gnu.org/licenses/>.                                      *
22
# *                                                                         *
23
# ***************************************************************************
24

25
""" Worker thread classes for Addon Manager startup """
26
import datetime
27
import hashlib
28
import json
29
import os
30
import queue
31
import re
32
import shutil
33
import stat
34
import threading
35
import time
36
from typing import List
37

38
from PySide import QtCore
39

40
import FreeCAD
41
import addonmanager_utilities as utils
42
from addonmanager_macro import Macro
43
from Addon import Addon
44
from AddonStats import AddonStats
45
import NetworkManager
46
from addonmanager_git import initialize_git, GitFailed
47
from addonmanager_metadata import MetadataReader, get_branch_from_metadata
48

49
translate = FreeCAD.Qt.translate
50

51
# Workers only have one public method by design
52
# pylint: disable=c-extension-no-member,too-few-public-methods,too-many-instance-attributes
53

54

55
class CreateAddonListWorker(QtCore.QThread):
56
    """This worker updates the list of available workbenches, emitting an "addon_repo"
57
    signal for each Addon as they are processed."""
58

59
    status_message = QtCore.Signal(str)
60
    addon_repo = QtCore.Signal(object)
61

62
    def __init__(self):
63
        QtCore.QThread.__init__(self)
64

65
        # reject_listed addons
66
        self.macros_reject_list = []
67
        self.mod_reject_list = []
68

69
        # These addons will print an additional message informing the user
70
        self.obsolete = []
71

72
        # These addons will print an additional message informing the user Python2 only
73
        self.py2only = []
74

75
        self.package_names = []
76
        self.moddir = os.path.join(FreeCAD.getUserAppDataDir(), "Mod")
77
        self.current_thread = None
78

79
        self.git_manager = initialize_git()
80

81
    def run(self):
82
        "populates the list of addons"
83

84
        self.current_thread = QtCore.QThread.currentThread()
85
        try:
86
            self._get_freecad_addon_repo_data()
87
        except ConnectionError:
88
            return
89
        self._get_custom_addons()
90
        self._get_official_addons()
91
        self._retrieve_macros_from_git()
92
        self._retrieve_macros_from_wiki()
93

94
    def _get_freecad_addon_repo_data(self):
95
        # update info lists
96
        p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(
97
            "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json", 5000
98
        )
99
        if p:
100
            p = p.data().decode("utf8")
101
            j = json.loads(p)
102
            if "obsolete" in j and "Mod" in j["obsolete"]:
103
                self.obsolete = j["obsolete"]["Mod"]
104

105
            if "blacklisted" in j and "Macro" in j["blacklisted"]:
106
                self.macros_reject_list = j["blacklisted"]["Macro"]
107

108
            if "blacklisted" in j and "Mod" in j["blacklisted"]:
109
                self.mod_reject_list = j["blacklisted"]["Mod"]
110

111
            if "py2only" in j and "Mod" in j["py2only"]:
112
                self.py2only = j["py2only"]["Mod"]
113

114
            if "deprecated" in j:
115
                self._process_deprecated(j["deprecated"])
116

117
        else:
118
            message = translate(
119
                "AddonsInstaller",
120
                "Failed to connect to GitHub. Check your connection and proxy settings.",
121
            )
122
            FreeCAD.Console.PrintError(message + "\n")
123
            self.status_message.emit(message)
124
            raise ConnectionError
125

126
    def _process_deprecated(self, deprecated_addons):
127
        """Parse the section on deprecated addons"""
128

129
        fc_major = int(FreeCAD.Version()[0])
130
        fc_minor = int(FreeCAD.Version()[1])
131
        for item in deprecated_addons:
132
            if "as_of" in item and "name" in item:
133
                try:
134
                    version_components = item["as_of"].split(".")
135
                    major = int(version_components[0])
136
                    if len(version_components) > 1:
137
                        minor = int(version_components[1])
138
                    else:
139
                        minor = 0
140
                    if major < fc_major or (major == fc_major and minor <= fc_minor):
141
                        if "kind" not in item or item["kind"] == "mod":
142
                            self.obsolete.append(item["name"])
143
                        elif item["kind"] == "macro":
144
                            self.macros_reject_list.append(item["name"])
145
                        else:
146
                            FreeCAD.Console.PrintMessage(
147
                                f'Unrecognized Addon kind {item["kind"]} in deprecation list.'
148
                            )
149
                except ValueError:
150
                    FreeCAD.Console.PrintMessage(
151
                        f"Failed to parse version from {item['name']}, version {item['as_of']}"
152
                    )
153

154
    def _get_custom_addons(self):
155

156
        # querying custom addons first
157
        addon_list = (
158
            FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
159
            .GetString("CustomRepositories", "")
160
            .split("\n")
161
        )
162
        custom_addons = []
163
        for addon in addon_list:
164
            if " " in addon:
165
                addon_and_branch = addon.split(" ")
166
                custom_addons.append({"url": addon_and_branch[0], "branch": addon_and_branch[1]})
167
            else:
168
                custom_addons.append({"url": addon, "branch": "master"})
169
        for addon in custom_addons:
170
            if self.current_thread.isInterruptionRequested():
171
                return
172
            if addon and addon["url"]:
173
                if addon["url"][-1] == "/":
174
                    addon["url"] = addon["url"][0:-1]  # Strip trailing slash
175
                addon["url"] = addon["url"].split(".git")[0]  # Remove .git
176
                name = addon["url"].split("/")[-1]
177
                if name in self.package_names:
178
                    # We already have something with this name, skip this one
179
                    FreeCAD.Console.PrintWarning(
180
                        translate("AddonsInstaller", "WARNING: Duplicate addon {} ignored").format(
181
                            name
182
                        )
183
                    )
184
                    continue
185
                FreeCAD.Console.PrintLog(
186
                    f"Adding custom location {addon['url']} with branch {addon['branch']}\n"
187
                )
188
                self.package_names.append(name)
189
                addondir = os.path.join(self.moddir, name)
190
                if os.path.exists(addondir) and os.listdir(addondir):
191
                    state = Addon.Status.UNCHECKED
192
                else:
193
                    state = Addon.Status.NOT_INSTALLED
194
                repo = Addon(name, addon["url"], state, addon["branch"])
195
                md_file = os.path.join(addondir, "package.xml")
196
                if os.path.isfile(md_file):
197
                    repo.installed_metadata = MetadataReader.from_file(md_file)
198
                    repo.installed_version = repo.installed_metadata.version
199
                    repo.updated_timestamp = os.path.getmtime(md_file)
200
                    repo.verify_url_and_branch(addon["url"], addon["branch"])
201

202
                self.addon_repo.emit(repo)
203

204
    def _get_official_addons(self):
205
        # querying official addons
206
        p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(
207
            "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules", 5000
208
        )
209
        if not p:
210
            return
211
        p = p.data().decode("utf8")
212
        p = re.findall(
213
            (
214
                r'(?m)\[submodule\s*"(?P<name>.*)"\]\s*'
215
                r"path\s*=\s*(?P<path>.+)\s*"
216
                r"url\s*=\s*(?P<url>https?://.*)\s*"
217
                r"(branch\s*=\s*(?P<branch>[^\s]*)\s*)?"
218
            ),
219
            p,
220
        )
221
        for name, _, url, _, branch in p:
222
            if self.current_thread.isInterruptionRequested():
223
                return
224
            if name in self.package_names:
225
                # We already have something with this name, skip this one
226
                continue
227
            self.package_names.append(name)
228
            if branch is None or len(branch) == 0:
229
                branch = "master"
230
            url = url.split(".git")[0]
231
            addondir = os.path.join(self.moddir, name)
232
            if os.path.exists(addondir) and os.listdir(addondir):
233
                # make sure the folder exists and it contains files!
234
                state = Addon.Status.UNCHECKED
235
            else:
236
                state = Addon.Status.NOT_INSTALLED
237
            repo = Addon(name, url, state, branch)
238
            md_file = os.path.join(addondir, "package.xml")
239
            if os.path.isfile(md_file):
240
                repo.installed_metadata = MetadataReader.from_file(md_file)
241
                repo.installed_version = repo.installed_metadata.version
242
                repo.updated_timestamp = os.path.getmtime(md_file)
243
                repo.verify_url_and_branch(url, branch)
244

245
            if name in self.py2only:
246
                repo.python2 = True
247
            if name in self.mod_reject_list:
248
                repo.rejected = True
249
            if name in self.obsolete:
250
                repo.obsolete = True
251
            self.addon_repo.emit(repo)
252

253
            self.status_message.emit(translate("AddonsInstaller", "Workbenches list was updated."))
254

255
    def _retrieve_macros_from_git(self):
256
        """Retrieve macros from FreeCAD-macros.git
257

258
        Emits a signal for each macro in
259
        https://github.com/FreeCAD/FreeCAD-macros.git
260
        """
261

262
        macro_cache_location = utils.get_cache_file_name("Macros")
263

264
        if not self.git_manager:
265
            message = translate(
266
                "AddonsInstaller",
267
                "Git is disabled, skipping git macros",
268
            )
269
            self.status_message.emit(message)
270
            FreeCAD.Console.PrintWarning(message + "\n")
271
            return
272

273
        update_succeeded = self._update_local_git_repo()
274
        if not update_succeeded:
275
            return
276

277
        n_files = 0
278
        for _, _, filenames in os.walk(macro_cache_location):
279
            n_files += len(filenames)
280
        counter = 0
281
        for dirpath, _, filenames in os.walk(macro_cache_location):
282
            counter += 1
283
            if self.current_thread.isInterruptionRequested():
284
                return
285
            if ".git" in dirpath:
286
                continue
287
            for filename in filenames:
288
                if self.current_thread.isInterruptionRequested():
289
                    return
290
                if filename.lower().endswith(".fcmacro"):
291
                    macro = Macro(filename[:-8])  # Remove ".FCMacro".
292
                    if macro.name in self.package_names:
293
                        FreeCAD.Console.PrintLog(
294
                            f"Ignoring second macro named {macro.name} (found on git)\n"
295
                        )
296
                        continue  # We already have a macro with this name
297
                    self.package_names.append(macro.name)
298
                    macro.on_git = True
299
                    macro.src_filename = os.path.join(dirpath, filename)
300
                    macro.fill_details_from_file(macro.src_filename)
301
                    repo = Addon.from_macro(macro)
302
                    FreeCAD.Console.PrintLog(f"Found macro {repo.name}\n")
303
                    repo.url = "https://github.com/FreeCAD/FreeCAD-macros.git"
304
                    utils.update_macro_installation_details(repo)
305
                    self.addon_repo.emit(repo)
306

307
    def _update_local_git_repo(self) -> bool:
308
        macro_cache_location = utils.get_cache_file_name("Macros")
309
        try:
310
            if os.path.exists(macro_cache_location):
311
                if not os.path.exists(os.path.join(macro_cache_location, ".git")):
312
                    FreeCAD.Console.PrintWarning(
313
                        translate(
314
                            "AddonsInstaller",
315
                            "Attempting to change non-git Macro setup to use git\n",
316
                        )
317
                    )
318
                    self.git_manager.repair(
319
                        "https://github.com/FreeCAD/FreeCAD-macros.git",
320
                        macro_cache_location,
321
                    )
322
                self.git_manager.update(macro_cache_location)
323
            else:
324
                self.git_manager.clone(
325
                    "https://github.com/FreeCAD/FreeCAD-macros.git",
326
                    macro_cache_location,
327
                )
328
        except GitFailed as e:
329
            FreeCAD.Console.PrintMessage(
330
                translate(
331
                    "AddonsInstaller",
332
                    "An error occurred updating macros from GitHub, trying clean checkout...",
333
                )
334
                + f":\n{e}\n"
335
            )
336
            FreeCAD.Console.PrintMessage(f"{macro_cache_location}\n")
337
            FreeCAD.Console.PrintMessage(
338
                translate("AddonsInstaller", "Attempting to do a clean checkout...") + "\n"
339
            )
340
            try:
341
                os.chdir(
342
                    os.path.join(macro_cache_location, "..")
343
                )  # Make sure we are not IN this directory
344
                shutil.rmtree(macro_cache_location, onerror=self._remove_readonly)
345
                self.git_manager.clone(
346
                    "https://github.com/FreeCAD/FreeCAD-macros.git",
347
                    macro_cache_location,
348
                )
349
                FreeCAD.Console.PrintMessage(
350
                    translate("AddonsInstaller", "Clean checkout succeeded") + "\n"
351
                )
352
            except GitFailed as e2:
353
                # The Qt Python translation extractor doesn't support splitting this string (yet)
354
                # pylint: disable=line-too-long
355
                FreeCAD.Console.PrintWarning(
356
                    translate(
357
                        "AddonsInstaller",
358
                        "Failed to update macros from GitHub -- try clearing the Addon Manager's cache.",
359
                    )
360
                    + f":\n{str(e2)}\n"
361
                )
362
                return False
363
        return True
364

365
    def _retrieve_macros_from_wiki(self):
366
        """Retrieve macros from the wiki
367

368
        Read the wiki and emit a signal for each found macro.
369
        Reads only the page https://wiki.freecad.org/Macros_recipes
370
        """
371

372
        p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(
373
            "https://wiki.freecad.org/Macros_recipes", 5000
374
        )
375
        if not p:
376
            # The Qt Python translation extractor doesn't support splitting this string (yet)
377
            # pylint: disable=line-too-long
378
            FreeCAD.Console.PrintWarning(
379
                translate(
380
                    "AddonsInstaller",
381
                    "Error connecting to the Wiki, FreeCAD cannot retrieve the Wiki macro list at this time",
382
                )
383
                + "\n"
384
            )
385
            return
386
        p = p.data().decode("utf8")
387
        macros = re.findall('title="(Macro.*?)"', p)
388
        macros = [mac for mac in macros if "translated" not in mac]
389
        macro_names = []
390
        for _, mac in enumerate(macros):
391
            if self.current_thread.isInterruptionRequested():
392
                return
393
            macname = mac[6:]  # Remove "Macro ".
394
            macname = macname.replace("&amp;", "&")
395
            if not macname:
396
                continue
397
            if (
398
                (macname not in self.macros_reject_list)
399
                and ("recipes" not in macname.lower())
400
                and (macname not in macro_names)
401
            ):
402
                macro_names.append(macname)
403
                macro = Macro(macname)
404
                if macro.name in self.package_names:
405
                    FreeCAD.Console.PrintLog(
406
                        f"Ignoring second macro named {macro.name} (found on wiki)\n"
407
                    )
408
                    continue  # We already have a macro with this name
409
                self.package_names.append(macro.name)
410
                macro.on_wiki = True
411
                macro.parsed = False
412
                repo = Addon.from_macro(macro)
413
                repo.url = "https://wiki.freecad.org/Macros_recipes"
414
                utils.update_macro_installation_details(repo)
415
                self.addon_repo.emit(repo)
416

417
    def _remove_readonly(self, func, path, _) -> None:
418
        """Remove a read-only file."""
419

420
        os.chmod(path, stat.S_IWRITE)
421
        func(path)
422

423

424
class LoadPackagesFromCacheWorker(QtCore.QThread):
425
    """A subthread worker that loads package information from its cache file."""
426

427
    addon_repo = QtCore.Signal(object)
428

429
    def __init__(self, cache_file: str):
430
        QtCore.QThread.__init__(self)
431
        self.cache_file = cache_file
432
        self.metadata_cache_path = os.path.join(
433
            FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata"
434
        )
435

436
    def override_metadata_cache_path(self, path):
437
        """For testing purposes, override the location to fetch the package metadata from."""
438
        self.metadata_cache_path = path
439

440
    def run(self):
441
        """Rarely called directly: create an instance and call start() on it instead to
442
        launch in a new thread"""
443
        with open(self.cache_file, encoding="utf-8") as f:
444
            data = f.read()
445
            if data:
446
                dict_data = json.loads(data)
447
                for item in dict_data.values():
448
                    if QtCore.QThread.currentThread().isInterruptionRequested():
449
                        return
450
                    repo = Addon.from_cache(item)
451
                    repo_metadata_cache_path = os.path.join(
452
                        self.metadata_cache_path, repo.name, "package.xml"
453
                    )
454
                    if os.path.isfile(repo_metadata_cache_path):
455
                        try:
456
                            repo.load_metadata_file(repo_metadata_cache_path)
457
                        except Exception as e:
458
                            FreeCAD.Console.PrintLog(f"Failed loading {repo_metadata_cache_path}\n")
459
                            FreeCAD.Console.PrintLog(str(e) + "\n")
460
                    self.addon_repo.emit(repo)
461

462

463
class LoadMacrosFromCacheWorker(QtCore.QThread):
464
    """A worker object to load macros from a cache file"""
465

466
    add_macro_signal = QtCore.Signal(object)
467

468
    def __init__(self, cache_file: str):
469
        QtCore.QThread.__init__(self)
470
        self.cache_file = cache_file
471

472
    def run(self):
473
        """Rarely called directly: create an instance and call start() on it instead to
474
        launch in a new thread"""
475

476
        with open(self.cache_file, encoding="utf-8") as f:
477
            data = f.read()
478
            dict_data = json.loads(data)
479
            for item in dict_data:
480
                if QtCore.QThread.currentThread().isInterruptionRequested():
481
                    return
482
                new_macro = Macro.from_cache(item)
483
                repo = Addon.from_macro(new_macro)
484
                utils.update_macro_installation_details(repo)
485
                self.add_macro_signal.emit(repo)
486

487

488
class CheckSingleUpdateWorker(QtCore.QObject):
489
    """This worker is a little different from the others: the actual recommended way of
490
    running in a QThread is to make a worker object that gets moved into the thread."""
491

492
    update_status = QtCore.Signal(int)
493

494
    def __init__(self, repo: Addon, parent: QtCore.QObject = None):
495
        super().__init__(parent)
496
        self.repo = repo
497

498
    def do_work(self):
499
        """Use the UpdateChecker class to do the work of this function, depending on the
500
        type of Addon"""
501

502
        checker = UpdateChecker()
503
        if self.repo.repo_type == Addon.Kind.WORKBENCH:
504
            checker.check_workbench(self.repo)
505
        elif self.repo.repo_type == Addon.Kind.MACRO:
506
            checker.check_macro(self.repo)
507
        elif self.repo.repo_type == Addon.Kind.PACKAGE:
508
            checker.check_package(self.repo)
509

510
        self.update_status.emit(self.repo.update_status)
511

512

513
class CheckWorkbenchesForUpdatesWorker(QtCore.QThread):
514
    """This worker checks for available updates for all workbenches"""
515

516
    update_status = QtCore.Signal(Addon)
517
    progress_made = QtCore.Signal(int, int)
518

519
    def __init__(self, repos: List[Addon]):
520

521
        QtCore.QThread.__init__(self)
522
        self.repos = repos
523
        self.current_thread = None
524
        self.basedir = FreeCAD.getUserAppDataDir()
525
        self.moddir = os.path.join(self.basedir, "Mod")
526

527
    def run(self):
528
        """Rarely called directly: create an instance and call start() on it instead to
529
        launch in a new thread"""
530

531
        self.current_thread = QtCore.QThread.currentThread()
532
        checker = UpdateChecker()
533
        count = 1
534
        for repo in self.repos:
535
            if self.current_thread.isInterruptionRequested():
536
                return
537
            self.progress_made.emit(count, len(self.repos))
538
            count += 1
539
            if repo.status() == Addon.Status.UNCHECKED:
540
                if repo.repo_type == Addon.Kind.WORKBENCH:
541
                    checker.check_workbench(repo)
542
                    self.update_status.emit(repo)
543
                elif repo.repo_type == Addon.Kind.MACRO:
544
                    checker.check_macro(repo)
545
                    self.update_status.emit(repo)
546
                elif repo.repo_type == Addon.Kind.PACKAGE:
547
                    checker.check_package(repo)
548
                    self.update_status.emit(repo)
549

550

551
class UpdateChecker:
552
    """A utility class used by the CheckWorkbenchesForUpdatesWorker class. Each function is
553
    designed for a specific Addon type, and modifies the passed-in Addon with the determined
554
    update status."""
555

556
    def __init__(self):
557
        self.basedir = FreeCAD.getUserAppDataDir()
558
        self.moddir = os.path.join(self.basedir, "Mod")
559
        self.git_manager = initialize_git()
560

561
    def override_mod_directory(self, moddir):
562
        """Primarily for use when testing, sets an alternate directory to use for mods"""
563
        self.moddir = moddir
564

565
    def check_workbench(self, wb):
566
        """Given a workbench Addon wb, check it for updates using git. If git is not
567
        available, does nothing."""
568
        if not self.git_manager:
569
            wb.set_status(Addon.Status.CANNOT_CHECK)
570
            return
571
        clonedir = os.path.join(self.moddir, wb.name)
572
        if os.path.exists(clonedir):
573
            # mark as already installed AND already checked for updates
574
            if not os.path.exists(os.path.join(clonedir, ".git")):
575
                with wb.git_lock:
576
                    self.git_manager.repair(wb.url, clonedir)
577
            with wb.git_lock:
578
                try:
579
                    status = self.git_manager.status(clonedir)
580
                    if "(no branch)" in self.git_manager.status(clonedir):
581
                        # By definition, in a detached-head state we cannot
582
                        # update, so don't even bother checking.
583
                        wb.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
584
                        wb.branch = self.git_manager.current_branch(clonedir)
585
                        return
586
                except GitFailed as e:
587
                    FreeCAD.Console.PrintWarning(
588
                        "AddonManager: "
589
                        + translate(
590
                            "AddonsInstaller",
591
                            "Unable to fetch git updates for workbench {}",
592
                        ).format(wb.name)
593
                        + "\n"
594
                    )
595
                    FreeCAD.Console.PrintWarning(str(e) + "\n")
596
                    wb.set_status(Addon.Status.CANNOT_CHECK)
597
                else:
598
                    try:
599
                        if self.git_manager.update_available(clonedir):
600
                            wb.set_status(Addon.Status.UPDATE_AVAILABLE)
601
                        else:
602
                            wb.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
603
                    except GitFailed:
604
                        FreeCAD.Console.PrintWarning(
605
                            translate("AddonsInstaller", "git status failed for {}").format(wb.name)
606
                            + "\n"
607
                        )
608
                        wb.set_status(Addon.Status.CANNOT_CHECK)
609

610
    def _branch_name_changed(self, package: Addon) -> bool:
611
        clone_dir = os.path.join(self.moddir, package.name)
612
        installed_metadata_file = os.path.join(clone_dir, "package.xml")
613
        if not os.path.isfile(installed_metadata_file):
614
            return False
615
        try:
616
            installed_metadata = MetadataReader.from_file(installed_metadata_file)
617
            installed_default_branch = get_branch_from_metadata(installed_metadata)
618
            remote_default_branch = get_branch_from_metadata(package.metadata)
619
            if installed_default_branch != remote_default_branch:
620
                return True
621
        except Exception:
622
            return False
623
        return False
624

625
    def check_package(self, package: Addon) -> None:
626
        """Given a packaged Addon package, check it for updates. If git is available that is
627
        used. If not, the package's metadata is examined, and if the metadata file has changed
628
        compared to the installed copy, an update is flagged. In addition, a change to the
629
        default branch name triggers an update."""
630

631
        clone_dir = self.moddir + os.sep + package.name
632
        if os.path.exists(clone_dir):
633

634
            # First, see if the branch name changed, which automatically triggers an update
635
            if self._branch_name_changed(package):
636
                package.set_status(Addon.Status.UPDATE_AVAILABLE)
637
                return
638

639
            # Next, try to just do a git-based update, which will give the most accurate results:
640
            if self.git_manager:
641
                self.check_workbench(package)
642
                if package.status() != Addon.Status.CANNOT_CHECK:
643
                    # It worked, just exit now
644
                    return
645

646
            # If we were unable to do a git-based update, try using the package.xml file instead:
647
            installed_metadata_file = os.path.join(clone_dir, "package.xml")
648
            if not os.path.isfile(installed_metadata_file):
649
                # If there is no package.xml file, then it's because the package author added it
650
                # after the last time the local installation was updated. By definition, then,
651
                # there is an update available, if only to download the new XML file.
652
                package.set_status(Addon.Status.UPDATE_AVAILABLE)
653
                package.installed_version = None
654
                return
655
            package.updated_timestamp = os.path.getmtime(installed_metadata_file)
656
            try:
657
                installed_metadata = MetadataReader.from_file(installed_metadata_file)
658
                package.installed_version = installed_metadata.version
659
                # Packages are considered up-to-date if the metadata version matches.
660
                # Authors should update their version string when they want the addon
661
                # manager to alert users of a new version.
662
                if package.metadata.version != installed_metadata.version:
663
                    package.set_status(Addon.Status.UPDATE_AVAILABLE)
664
                else:
665
                    package.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
666
            except Exception:
667
                FreeCAD.Console.PrintWarning(
668
                    translate(
669
                        "AddonsInstaller",
670
                        "Failed to read metadata from {name}",
671
                    ).format(name=installed_metadata_file)
672
                    + "\n"
673
                )
674
                package.set_status(Addon.Status.CANNOT_CHECK)
675

676
    def check_macro(self, macro_wrapper: Addon) -> None:
677
        """Check to see if the online copy of the macro's code differs from the local copy."""
678

679
        # Make sure this macro has its code downloaded:
680
        try:
681
            if not macro_wrapper.macro.parsed and macro_wrapper.macro.on_git:
682
                macro_wrapper.macro.fill_details_from_file(macro_wrapper.macro.src_filename)
683
            elif not macro_wrapper.macro.parsed and macro_wrapper.macro.on_wiki:
684
                mac = macro_wrapper.macro.name.replace(" ", "_")
685
                mac = mac.replace("&", "%26")
686
                mac = mac.replace("+", "%2B")
687
                url = "https://wiki.freecad.org/Macro_" + mac
688
                macro_wrapper.macro.fill_details_from_wiki(url)
689
        except Exception:
690
            FreeCAD.Console.PrintWarning(
691
                translate(
692
                    "AddonsInstaller",
693
                    "Failed to fetch code for macro '{name}'",
694
                ).format(name=macro_wrapper.macro.name)
695
                + "\n"
696
            )
697
            macro_wrapper.set_status(Addon.Status.CANNOT_CHECK)
698
            return
699

700
        hasher1 = hashlib.sha1()
701
        hasher2 = hashlib.sha1()
702
        hasher1.update(macro_wrapper.macro.code.encode("utf-8"))
703
        new_sha1 = hasher1.hexdigest()
704
        test_file_one = os.path.join(FreeCAD.getUserMacroDir(True), macro_wrapper.macro.filename)
705
        test_file_two = os.path.join(
706
            FreeCAD.getUserMacroDir(True), "Macro_" + macro_wrapper.macro.filename
707
        )
708
        if os.path.exists(test_file_one):
709
            with open(test_file_one, "rb") as f:
710
                contents = f.read()
711
                hasher2.update(contents)
712
                old_sha1 = hasher2.hexdigest()
713
        elif os.path.exists(test_file_two):
714
            with open(test_file_two, "rb") as f:
715
                contents = f.read()
716
                hasher2.update(contents)
717
                old_sha1 = hasher2.hexdigest()
718
        else:
719
            return
720
        if new_sha1 == old_sha1:
721
            macro_wrapper.set_status(Addon.Status.NO_UPDATE_AVAILABLE)
722
        else:
723
            macro_wrapper.set_status(Addon.Status.UPDATE_AVAILABLE)
724

725

726
class CacheMacroCodeWorker(QtCore.QThread):
727
    """Download and cache the macro code, and parse its internal metadata"""
728

729
    status_message = QtCore.Signal(str)
730
    update_macro = QtCore.Signal(Addon)
731
    progress_made = QtCore.Signal(int, int)
732

733
    def __init__(self, repos: List[Addon]) -> None:
734
        QtCore.QThread.__init__(self)
735
        self.repos = repos
736
        self.workers = []
737
        self.terminators = []
738
        self.lock = threading.Lock()
739
        self.failed = []
740
        self.counter = 0
741
        self.repo_queue = None
742

743
    def run(self):
744
        """Rarely called directly: create an instance and call start() on it instead to
745
        launch in a new thread"""
746

747
        self.status_message.emit(translate("AddonsInstaller", "Caching macro code..."))
748

749
        self.repo_queue = queue.Queue()
750
        num_macros = 0
751
        for repo in self.repos:
752
            if repo.macro is not None:
753
                self.repo_queue.put(repo)
754
                num_macros += 1
755

756
        interrupted = self._process_queue(num_macros)
757
        if interrupted:
758
            return
759

760
        # Make sure all of our child threads have fully exited:
761
        for worker in self.workers:
762
            worker.wait(50)
763
            if not worker.isFinished():
764
                # The Qt Python translation extractor doesn't support splitting this string (yet)
765
                # pylint: disable=line-too-long
766
                FreeCAD.Console.PrintError(
767
                    translate(
768
                        "AddonsInstaller",
769
                        "Addon Manager: a worker process failed to complete while fetching {name}",
770
                    ).format(name=worker.macro.name)
771
                    + "\n"
772
                )
773

774
        self.repo_queue.join()
775
        for terminator in self.terminators:
776
            if terminator and terminator.isActive():
777
                terminator.stop()
778

779
        if len(self.failed) > 0:
780
            num_failed = len(self.failed)
781
            FreeCAD.Console.PrintWarning(
782
                translate(
783
                    "AddonsInstaller",
784
                    "Out of {num_macros} macros, {num_failed} timed out while processing",
785
                ).format(num_macros=num_macros, num_failed=num_failed)
786
            )
787

788
    def _process_queue(self, num_macros) -> bool:
789
        """Spools up six network connections and downloads the macro code. Returns True if
790
        it was interrupted by user request, or False if it ran to completion."""
791

792
        # Emulate QNetworkAccessManager and spool up six connections:
793
        for _ in range(6):
794
            self.update_and_advance(None)
795

796
        current_thread = QtCore.QThread.currentThread()
797
        while True:
798
            if current_thread.isInterruptionRequested():
799
                for worker in self.workers:
800
                    worker.blockSignals(True)
801
                    worker.requestInterruption()
802
                    if not worker.wait(100):
803
                        FreeCAD.Console.PrintWarning(
804
                            translate(
805
                                "AddonsInstaller",
806
                                "Addon Manager: a worker process failed to halt ({name})",
807
                            ).format(name=worker.macro.name)
808
                            + "\n"
809
                        )
810
                return True
811
            # Ensure our signals propagate out by running an internal thread-local event loop
812
            QtCore.QCoreApplication.processEvents()
813
            with self.lock:
814
                if self.counter >= num_macros:
815
                    break
816
            time.sleep(0.1)
817
        return False
818

819
    def update_and_advance(self, repo: Addon) -> None:
820
        """Emit the updated signal and launch the next item from the queue."""
821
        if repo is not None:
822
            if repo.macro.name not in self.failed:
823
                self.update_macro.emit(repo)
824
            self.repo_queue.task_done()
825
            with self.lock:
826
                self.counter += 1
827

828
        if QtCore.QThread.currentThread().isInterruptionRequested():
829
            return
830

831
        self.progress_made.emit(len(self.repos) - self.repo_queue.qsize(), len(self.repos))
832

833
        try:
834
            next_repo = self.repo_queue.get_nowait()
835
            worker = GetMacroDetailsWorker(next_repo)
836
            worker.finished.connect(lambda: self.update_and_advance(next_repo))
837
            with self.lock:
838
                self.workers.append(worker)
839
                self.terminators.append(
840
                    QtCore.QTimer.singleShot(10000, lambda: self.terminate(worker))
841
                )
842
            self.status_message.emit(
843
                translate(
844
                    "AddonsInstaller",
845
                    "Getting metadata from macro {}",
846
                ).format(next_repo.macro.name)
847
            )
848
            worker.start()
849
        except queue.Empty:
850
            pass
851

852
    def terminate(self, worker) -> None:
853
        """Shut down all running workers and exit the thread"""
854
        if not worker.isFinished():
855
            macro_name = worker.macro.name
856
            FreeCAD.Console.PrintWarning(
857
                translate(
858
                    "AddonsInstaller",
859
                    "Timeout while fetching metadata for macro {}",
860
                ).format(macro_name)
861
                + "\n"
862
            )
863
            # worker.blockSignals(True)
864
            worker.requestInterruption()
865
            worker.wait(100)
866
            if worker.isRunning():
867
                FreeCAD.Console.PrintError(
868
                    translate(
869
                        "AddonsInstaller",
870
                        "Failed to kill process for macro {}!\n",
871
                    ).format(macro_name)
872
                )
873
            with self.lock:
874
                self.failed.append(macro_name)
875

876

877
class GetMacroDetailsWorker(QtCore.QThread):
878
    """Retrieve the macro details for a macro"""
879

880
    status_message = QtCore.Signal(str)
881
    readme_updated = QtCore.Signal(str)
882

883
    def __init__(self, repo):
884

885
        QtCore.QThread.__init__(self)
886
        self.macro = repo.macro
887

888
    def run(self):
889
        """Rarely called directly: create an instance and call start() on it instead to
890
        launch in a new thread"""
891

892
        self.status_message.emit(translate("AddonsInstaller", "Retrieving macro description..."))
893
        if not self.macro.parsed and self.macro.on_git:
894
            self.status_message.emit(translate("AddonsInstaller", "Retrieving info from git"))
895
            self.macro.fill_details_from_file(self.macro.src_filename)
896
        if not self.macro.parsed and self.macro.on_wiki:
897
            self.status_message.emit(translate("AddonsInstaller", "Retrieving info from wiki"))
898
            mac = self.macro.name.replace(" ", "_")
899
            mac = mac.replace("&", "%26")
900
            mac = mac.replace("+", "%2B")
901
            url = "https://wiki.freecad.org/Macro_" + mac
902
            self.macro.fill_details_from_wiki(url)
903
        message = (
904
            "<h1>"
905
            + self.macro.name
906
            + "</h1>"
907
            + self.macro.desc
908
            + '<br/><br/>Macro location: <a href="'
909
            + self.macro.url
910
            + '">'
911
            + self.macro.url
912
            + "</a>"
913
        )
914
        if QtCore.QThread.currentThread().isInterruptionRequested():
915
            return
916
        self.readme_updated.emit(message)
917

918

919
class GetBasicAddonStatsWorker(QtCore.QThread):
920
    """Fetch data from an addon stats repository."""
921

922
    update_addon_stats = QtCore.Signal(Addon)
923

924
    def __init__(self, url: str, addons: List[Addon], parent: QtCore.QObject = None):
925
        super().__init__(parent)
926
        self.url = url
927
        self.addons = addons
928

929
    def run(self):
930
        """Fetch the remote data and load it into the addons"""
931

932
        fetch_result = NetworkManager.AM_NETWORK_MANAGER.blocking_get(self.url, 5000)
933
        if fetch_result is None:
934
            FreeCAD.Console.PrintError(
935
                translate(
936
                    "AddonsInstaller",
937
                    "Failed to get Addon statistics from {} -- only sorting alphabetically will be accurate\n",
938
                ).format(self.url)
939
            )
940
            return
941
        text_result = fetch_result.data().decode("utf8")
942
        json_result = json.loads(text_result)
943

944
        for addon in self.addons:
945
            if addon.url in json_result:
946
                addon.stats = AddonStats.from_json(json_result[addon.url])
947
                self.update_addon_stats.emit(addon)
948

949

950
class GetAddonScoreWorker(QtCore.QThread):
951
    """Fetch data from an addon score file."""
952

953
    update_addon_score = QtCore.Signal(Addon)
954

955
    def __init__(self, url: str, addons: List[Addon], parent: QtCore.QObject = None):
956
        super().__init__(parent)
957
        self.url = url
958
        self.addons = addons
959

960
    def run(self):
961
        """Fetch the remote data and load it into the addons"""
962

963
        if self.url != "TEST":
964
            fetch_result = NetworkManager.AM_NETWORK_MANAGER.blocking_get(self.url, 5000)
965
            if fetch_result is None:
966
                FreeCAD.Console.PrintError(
967
                    translate(
968
                        "AddonsInstaller",
969
                        "Failed to get Addon score from '{}' -- sorting by score will fail\n",
970
                    ).format(self.url)
971
                )
972
                return
973
            text_result = fetch_result.data().decode("utf8")
974
            json_result = json.loads(text_result)
975
        else:
976
            FreeCAD.Console.PrintWarning("Running score generation in TEST mode...\n")
977
            json_result = {}
978
            for addon in self.addons:
979
                if addon.macro:
980
                    json_result[addon.name] = len(addon.macro.comment) if addon.macro.comment else 0
981
                else:
982
                    json_result[addon.url] = len(addon.description) if addon.description else 0
983

984
        for addon in self.addons:
985
            score = None
986
            if addon.url in json_result:
987
                score = json_result[addon.url]
988
            elif addon.name in json_result:
989
                score = json_result[addon.name]
990
            if score is not None:
991
                try:
992
                    addon.score = int(score)
993
                    self.update_addon_score.emit(addon)
994
                except (ValueError, OverflowError):
995
                    FreeCAD.Console.PrintLog(
996
                        f"Failed to convert score value '{score}' to an integer for addon {addon.name}"
997
                    )
998