FreeCAD

1
# SPDX-License-Identifier: LGPL-2.1-or-later
2
# ***************************************************************************
3
# *                                                                         *
4
# *   Copyright (c) 2022-2023 FreeCAD Project Association                   *
5
# *   Copyright (c) 2019 Yorik van Havre <yorik@uncreated.net>              *
6
# *                                                                         *
7
# *   This file is part of FreeCAD.                                         *
8
# *                                                                         *
9
# *   FreeCAD is free software: you can redistribute it and/or modify it    *
10
# *   under the terms of the GNU Lesser General Public License as           *
11
# *   published by the Free Software Foundation, either version 2.1 of the  *
12
# *   License, or (at your option) any later version.                       *
13
# *                                                                         *
14
# *   FreeCAD is distributed in the hope that it will be useful, but        *
15
# *   WITHOUT ANY WARRANTY; without even the implied warranty of            *
16
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU      *
17
# *   Lesser General Public License for more details.                       *
18
# *                                                                         *
19
# *   You should have received a copy of the GNU Lesser General Public      *
20
# *   License along with FreeCAD. If not, see                               *
21
# *   <https://www.gnu.org/licenses/>.                                      *
22
# *                                                                         *
23
# ***************************************************************************
24

25
""" Worker thread classes for Addon Manager installation and removal """
26

27
# pylint: disable=c-extension-no-member,too-few-public-methods,too-many-instance-attributes
28

29
import io
30
import os
31
import queue
32
import shutil
33
import subprocess
34
import time
35
import zipfile
36
from typing import Dict, List
37
from enum import Enum, auto
38

39
from PySide import QtCore
40

41
import FreeCAD
42
import addonmanager_utilities as utils
43
from addonmanager_metadata import MetadataReader
44
from Addon import Addon
45
import NetworkManager
46

47
translate = FreeCAD.Qt.translate
48

49
#  @package AddonManager_workers
50
#  \ingroup ADDONMANAGER
51
#  \brief Multithread workers for the addon manager
52
#  @{
53

54

55
class UpdateMetadataCacheWorker(QtCore.QThread):
56
    """Scan through all available packages and see if our local copy of package.xml needs to be
57
    updated"""
58

59
    status_message = QtCore.Signal(str)
60
    progress_made = QtCore.Signal(int, int)
61
    package_updated = QtCore.Signal(Addon)
62

63
    class RequestType(Enum):
64
        """The type of item being downloaded."""
65

66
        PACKAGE_XML = auto()
67
        METADATA_TXT = auto()
68
        REQUIREMENTS_TXT = auto()
69
        ICON = auto()
70

71
    def __init__(self, repos):
72

73
        QtCore.QThread.__init__(self)
74
        self.repos = repos
75
        self.requests: Dict[int, (Addon, UpdateMetadataCacheWorker.RequestType)] = {}
76
        NetworkManager.AM_NETWORK_MANAGER.completed.connect(self.download_completed)
77
        self.requests_completed = 0
78
        self.total_requests = 0
79
        self.store = os.path.join(FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata")
80
        FreeCAD.Console.PrintLog(f"Storing Addon Manager cache data in {self.store}\n")
81
        self.updated_repos = set()
82

83
    def run(self):
84
        """Not usually called directly: instead, create an instance and call its
85
        start() function to spawn a new thread."""
86

87
        current_thread = QtCore.QThread.currentThread()
88

89
        for repo in self.repos:
90
            if not repo.macro and repo.url and utils.recognized_git_location(repo):
91
                # package.xml
92
                index = NetworkManager.AM_NETWORK_MANAGER.submit_unmonitored_get(
93
                    utils.construct_git_url(repo, "package.xml")
94
                )
95
                self.requests[index] = (
96
                    repo,
97
                    UpdateMetadataCacheWorker.RequestType.PACKAGE_XML,
98
                )
99
                self.total_requests += 1
100

101
                # metadata.txt
102
                index = NetworkManager.AM_NETWORK_MANAGER.submit_unmonitored_get(
103
                    utils.construct_git_url(repo, "metadata.txt")
104
                )
105
                self.requests[index] = (
106
                    repo,
107
                    UpdateMetadataCacheWorker.RequestType.METADATA_TXT,
108
                )
109
                self.total_requests += 1
110

111
                # requirements.txt
112
                index = NetworkManager.AM_NETWORK_MANAGER.submit_unmonitored_get(
113
                    utils.construct_git_url(repo, "requirements.txt")
114
                )
115
                self.requests[index] = (
116
                    repo,
117
                    UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT,
118
                )
119
                self.total_requests += 1
120

121
        while self.requests:
122
            if current_thread.isInterruptionRequested():
123
                for request in self.requests:
124
                    NetworkManager.AM_NETWORK_MANAGER.abort(request)
125
                return
126
            # 50 ms maximum between checks for interruption
127
            QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
128

129
        # This set contains one copy of each of the repos that got some kind of data in
130
        # this process. For those repos, tell the main Addon Manager code that it needs
131
        # to update its copy of the repo, and redraw its information.
132
        for repo in self.updated_repos:
133
            self.package_updated.emit(repo)
134

135
    def download_completed(self, index: int, code: int, data: QtCore.QByteArray) -> None:
136
        """Callback for handling a completed metadata file download."""
137
        if index in self.requests:
138
            self.requests_completed += 1
139
            self.progress_made.emit(self.requests_completed, self.total_requests)
140
            request = self.requests.pop(index)
141
            if code == 200:  # HTTP success
142
                self.updated_repos.add(request[0])  # mark this repo as updated
143
                if request[1] == UpdateMetadataCacheWorker.RequestType.PACKAGE_XML:
144
                    self.process_package_xml(request[0], data)
145
                elif request[1] == UpdateMetadataCacheWorker.RequestType.METADATA_TXT:
146
                    self.process_metadata_txt(request[0], data)
147
                elif request[1] == UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT:
148
                    self.process_requirements_txt(request[0], data)
149
                elif request[1] == UpdateMetadataCacheWorker.RequestType.ICON:
150
                    self.process_icon(request[0], data)
151

152
    def process_package_xml(self, repo: Addon, data: QtCore.QByteArray):
153
        """Process the package.xml metadata file"""
154
        repo.repo_type = Addon.Kind.PACKAGE  # By definition
155
        package_cache_directory = os.path.join(self.store, repo.name)
156
        if not os.path.exists(package_cache_directory):
157
            os.makedirs(package_cache_directory)
158
        new_xml_file = os.path.join(package_cache_directory, "package.xml")
159
        with open(new_xml_file, "wb") as f:
160
            f.write(data.data())
161
        metadata = MetadataReader.from_file(new_xml_file)
162
        repo.set_metadata(metadata)
163
        FreeCAD.Console.PrintLog(f"Downloaded package.xml for {repo.name}\n")
164
        self.status_message.emit(
165
            translate("AddonsInstaller", "Downloaded package.xml for {}").format(repo.name)
166
        )
167

168
        # Grab a new copy of the icon as well: we couldn't enqueue this earlier because
169
        # we didn't know the path to it, which is stored in the package.xml file.
170
        icon = repo.get_best_icon_relative_path()
171

172
        icon_url = utils.construct_git_url(repo, icon)
173
        index = NetworkManager.AM_NETWORK_MANAGER.submit_unmonitored_get(icon_url)
174
        self.requests[index] = (repo, UpdateMetadataCacheWorker.RequestType.ICON)
175
        self.total_requests += 1
176

177
    def _decode_data(self, byte_data, addon_name, file_name) -> str:
178
        """UTF-8 decode data, and print an error message if that fails"""
179

180
        # For review and debugging purposes, store the file locally
181
        package_cache_directory = os.path.join(self.store, addon_name)
182
        if not os.path.exists(package_cache_directory):
183
            os.makedirs(package_cache_directory)
184
        new_xml_file = os.path.join(package_cache_directory, file_name)
185
        with open(new_xml_file, "wb") as f:
186
            f.write(byte_data)
187

188
        f = ""
189
        try:
190
            f = byte_data.decode("utf-8")
191
        except UnicodeDecodeError as e:
192
            FreeCAD.Console.PrintWarning(
193
                translate(
194
                    "AddonsInstaller",
195
                    "Failed to decode {} file for Addon '{}'",
196
                ).format(file_name, addon_name)
197
                + "\n"
198
            )
199
            FreeCAD.Console.PrintWarning(str(e) + "\n")
200
            FreeCAD.Console.PrintWarning(
201
                translate(
202
                    "AddonsInstaller",
203
                    "Any dependency information in this file will be ignored",
204
                )
205
                + "\n"
206
            )
207
        return f
208

209
    def process_metadata_txt(self, repo: Addon, data: QtCore.QByteArray):
210
        """Process the metadata.txt metadata file"""
211
        self.status_message.emit(
212
            translate("AddonsInstaller", "Downloaded metadata.txt for {}").format(repo.display_name)
213
        )
214

215
        f = self._decode_data(data.data(), repo.name, "metadata.txt")
216
        lines = f.splitlines()
217
        for line in lines:
218
            if line.startswith("workbenches="):
219
                depswb = line.split("=")[1].split(",")
220
                for wb in depswb:
221
                    wb_name = wb.strip()
222
                    if wb_name:
223
                        repo.requires.add(wb_name)
224
                        FreeCAD.Console.PrintLog(
225
                            f"{repo.display_name} requires FreeCAD Addon '{wb_name}'\n"
226
                        )
227

228
            elif line.startswith("pylibs="):
229
                depspy = line.split("=")[1].split(",")
230
                for pl in depspy:
231
                    dep = pl.strip()
232
                    if dep:
233
                        repo.python_requires.add(dep)
234
                        FreeCAD.Console.PrintLog(
235
                            f"{repo.display_name} requires python package '{dep}'\n"
236
                        )
237

238
            elif line.startswith("optionalpylibs="):
239
                opspy = line.split("=")[1].split(",")
240
                for pl in opspy:
241
                    dep = pl.strip()
242
                    if dep:
243
                        repo.python_optional.add(dep)
244
                        FreeCAD.Console.PrintLog(
245
                            f"{repo.display_name} optionally imports python package"
246
                            + f" '{pl.strip()}'\n"
247
                        )
248

249
    def process_requirements_txt(self, repo: Addon, data: QtCore.QByteArray):
250
        """Process the requirements.txt metadata file"""
251
        self.status_message.emit(
252
            translate(
253
                "AddonsInstaller",
254
                "Downloaded requirements.txt for {}",
255
            ).format(repo.display_name)
256
        )
257

258
        f = self._decode_data(data.data(), repo.name, "requirements.txt")
259
        lines = f.splitlines()
260
        for line in lines:
261
            break_chars = " <>=~!+#"
262
            package = line
263
            for n, c in enumerate(line):
264
                if c in break_chars:
265
                    package = line[:n].strip()
266
                    break
267
            if package:
268
                repo.python_requires.add(package)
269

270
    def process_icon(self, repo: Addon, data: QtCore.QByteArray):
271
        """Convert icon data into a valid icon file and store it"""
272
        self.status_message.emit(
273
            translate("AddonsInstaller", "Downloaded icon for {}").format(repo.display_name)
274
        )
275
        cache_file = repo.get_cached_icon_filename()
276
        with open(cache_file, "wb") as icon_file:
277
            icon_file.write(data.data())
278
            repo.cached_icon_filename = cache_file
279

280

281
#  @}
282