FreeCAD

1
# SPDX-License-Identifier: LGPL-2.1-or-later
2
# ***************************************************************************
3
# *                                                                         *
4
# *   Copyright (c) 2022-2023 FreeCAD Project Association                   *
5
# *                                                                         *
6
# *   This file is part of FreeCAD.                                         *
7
# *                                                                         *
8
# *   FreeCAD is free software: you can redistribute it and/or modify it    *
9
# *   under the terms of the GNU Lesser General Public License as           *
10
# *   published by the Free Software Foundation, either version 2.1 of the  *
11
# *   License, or (at your option) any later version.                       *
12
# *                                                                         *
13
# *   FreeCAD is distributed in the hope that it will be useful, but        *
14
# *   WITHOUT ANY WARRANTY; without even the implied warranty of            *
15
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU      *
16
# *   Lesser General Public License for more details.                       *
17
# *                                                                         *
18
# *   You should have received a copy of the GNU Lesser General Public      *
19
# *   License along with FreeCAD. If not, see                               *
20
# *   <https://www.gnu.org/licenses/>.                                      *
21
# *                                                                         *
22
# ***************************************************************************
23

24
""" Provides classes and support functions for managing the automatically-installed
25
Python library dependencies. No support is provided for uninstalling those dependencies
26
because pip's uninstall function does not support the target directory argument. """
27

28
import json
29
import os
30
import platform
31
import shutil
32
import subprocess
33
import sys
34
from functools import partial
35
from typing import Dict, List, Tuple
36

37
import addonmanager_freecad_interface as fci
38

39
import FreeCAD
40
import FreeCADGui
41
from freecad.utils import get_python_exe
42
from PySide import QtCore, QtGui, QtWidgets
43

44
import addonmanager_utilities as utils
45

46
translate = FreeCAD.Qt.translate
47

48
# pylint: disable=too-few-public-methods
49

50

51
class PipFailed(Exception):
52
    """Exception thrown when pip times out or otherwise fails to return valid results"""
53

54

55
class CheckForPythonPackageUpdatesWorker(QtCore.QThread):
56
    """Perform non-blocking Python library update availability checking"""
57

58
    python_package_updates_available = QtCore.Signal()
59

60
    def __init__(self):
61
        QtCore.QThread.__init__(self)
62

63
    def run(self):
64
        """Usually not called directly: instead, instantiate this class and call its start()
65
        function in a parent thread. emits a python_package_updates_available signal if updates
66
        are available for any of the installed Python packages."""
67

68
        if check_for_python_package_updates():
69
            self.python_package_updates_available.emit()
70

71

72
def check_for_python_package_updates() -> bool:
73
    """Returns True if any of the Python packages installed into the AdditionalPythonPackages
74
    directory have updates available, or False if they are all up-to-date."""
75

76
    vendor_path = os.path.join(FreeCAD.getUserAppDataDir(), "AdditionalPythonPackages")
77
    package_counter = 0
78
    try:
79
        outdated_packages_stdout = call_pip(["list", "-o", "--path", vendor_path])
80
    except PipFailed as e:
81
        FreeCAD.Console.PrintError(str(e) + "\n")
82
        return False
83
    FreeCAD.Console.PrintLog("Output from pip -o:\n")
84
    for line in outdated_packages_stdout:
85
        if len(line) > 0:
86
            package_counter += 1
87
        FreeCAD.Console.PrintLog(f"  {line}\n")
88
    return package_counter > 0
89

90

91
def call_pip(args) -> List[str]:
92
    """Tries to locate the appropriate Python executable and run pip with version checking
93
    disabled. Fails if Python can't be found or if pip is not installed."""
94

95
    python_exe = get_python_exe()
96
    pip_failed = False
97
    if python_exe:
98
        call_args = [python_exe, "-m", "pip", "--disable-pip-version-check"]
99
        call_args.extend(args)
100
        proc = None
101
        try:
102
            proc = utils.run_interruptable_subprocess(call_args)
103
        except subprocess.CalledProcessError:
104
            pip_failed = True
105

106
        result = []
107
        if not pip_failed:
108
            data = proc.stdout
109
            result = data.split("\n")
110
        elif proc:
111
            raise PipFailed(proc.stderr)
112
        else:
113
            raise PipFailed("pip timed out")
114
    else:
115
        raise PipFailed("Could not locate Python executable on this system")
116
    return result
117

118

119
class PythonPackageManager:
120
    """A GUI-based pip interface allowing packages to be updated, either individually or all at
121
    once."""
122

123
    class PipRunner(QtCore.QObject):
124
        """Run pip in a separate thread so the UI doesn't block while it runs"""
125

126
        finished = QtCore.Signal()
127
        error = QtCore.Signal(str)
128

129
        def __init__(self, vendor_path, parent=None):
130
            super().__init__(parent)
131
            self.all_packages_stdout = []
132
            self.outdated_packages_stdout = []
133
            self.vendor_path = vendor_path
134
            self.package_list = {}
135

136
        def process(self):
137
            """Execute this object."""
138
            try:
139
                self.all_packages_stdout = call_pip(["list", "--path", self.vendor_path])
140
                self.outdated_packages_stdout = call_pip(["list", "-o", "--path", self.vendor_path])
141
            except PipFailed as e:
142
                FreeCAD.Console.PrintError(str(e) + "\n")
143
                self.error.emit(str(e))
144
            self.finished.emit()
145

146
    def __init__(self, addons):
147
        self.dlg = FreeCADGui.PySideUic.loadUi(
148
            os.path.join(os.path.dirname(__file__), "PythonDependencyUpdateDialog.ui")
149
        )
150
        self.addons = addons
151
        self.vendor_path = utils.get_pip_target_directory()
152
        self.worker_thread = None
153
        self.worker_object = None
154
        self.package_list = []
155

156
    def show(self):
157
        """Run the modal dialog"""
158

159
        known_python_versions = self.get_known_python_versions()
160
        if self._current_python_version_is_new() and known_python_versions:
161
            # pylint: disable=line-too-long
162
            result = QtWidgets.QMessageBox.question(
163
                None,
164
                translate("AddonsInstaller", "New Python Version Detected"),
165
                translate(
166
                    "AddonsInstaller",
167
                    "This appears to be the first time this version of Python has been used with the Addon Manager. "
168
                    "Would you like to install the same auto-installed dependencies for it?",
169
                ),
170
                QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
171
            )
172
            if result == QtWidgets.QMessageBox.Yes:
173
                self._reinstall_all_packages()
174

175
        self._add_current_python_version()
176
        self._create_list_from_pip()
177
        self.dlg.tableWidget.setSortingEnabled(False)
178
        self.dlg.labelInstallationPath.setText(self.vendor_path)
179
        self.dlg.exec()
180

181
    def _create_list_from_pip(self):
182
        """Uses pip and pip -o to generate a list of installed packages, and creates the user
183
        interface elements for those packages. Asynchronous, will complete AFTER the window is
184
        showing in most cases."""
185

186
        self.worker_thread = QtCore.QThread()
187
        self.worker_object = PythonPackageManager.PipRunner(self.vendor_path)
188
        self.worker_object.moveToThread(self.worker_thread)
189
        self.worker_object.finished.connect(self._worker_finished)
190
        self.worker_object.finished.connect(self.worker_thread.quit)
191
        self.worker_thread.started.connect(self.worker_object.process)
192
        self.worker_thread.start()
193

194
        self.dlg.tableWidget.setRowCount(1)
195
        self.dlg.tableWidget.setItem(
196
            0,
197
            0,
198
            QtWidgets.QTableWidgetItem(translate("AddonsInstaller", "Processing, please wait...")),
199
        )
200
        self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
201
            0, QtWidgets.QHeaderView.ResizeToContents
202
        )
203

204
    def _worker_finished(self):
205
        """Callback for when the worker process has completed"""
206
        all_packages_stdout = self.worker_object.all_packages_stdout
207
        outdated_packages_stdout = self.worker_object.outdated_packages_stdout
208

209
        self.package_list = self._parse_pip_list_output(
210
            all_packages_stdout, outdated_packages_stdout
211
        )
212
        self.dlg.buttonUpdateAll.clicked.connect(
213
            partial(self._update_all_packages, self.package_list)
214
        )
215

216
        self.dlg.tableWidget.setRowCount(len(self.package_list))
217
        updateButtons = []
218
        counter = 0
219
        update_counter = 0
220
        self.dlg.tableWidget.setSortingEnabled(False)
221
        for package_name, package_details in self.package_list.items():
222
            dependent_addons = self._get_dependent_addons(package_name)
223
            dependencies = []
224
            for addon in dependent_addons:
225
                if addon["optional"]:
226
                    dependencies.append(addon["name"] + "*")
227
                else:
228
                    dependencies.append(addon["name"])
229
            self.dlg.tableWidget.setItem(counter, 0, QtWidgets.QTableWidgetItem(package_name))
230
            self.dlg.tableWidget.setItem(
231
                counter,
232
                1,
233
                QtWidgets.QTableWidgetItem(package_details["installed_version"]),
234
            )
235
            self.dlg.tableWidget.setItem(
236
                counter,
237
                2,
238
                QtWidgets.QTableWidgetItem(package_details["available_version"]),
239
            )
240
            self.dlg.tableWidget.setItem(
241
                counter,
242
                3,
243
                QtWidgets.QTableWidgetItem(", ".join(dependencies)),
244
            )
245
            if len(package_details["available_version"]) > 0:
246
                updateButtons.append(QtWidgets.QPushButton(translate("AddonsInstaller", "Update")))
247
                updateButtons[-1].setIcon(QtGui.QIcon(":/icons/button_up.svg"))
248
                updateButtons[-1].clicked.connect(partial(self._update_package, package_name))
249
                self.dlg.tableWidget.setCellWidget(counter, 4, updateButtons[-1])
250
                update_counter += 1
251
            else:
252
                self.dlg.tableWidget.removeCellWidget(counter, 3)
253
            counter += 1
254
        self.dlg.tableWidget.setSortingEnabled(True)
255

256
        self.dlg.tableWidget.horizontalHeader().setStretchLastSection(False)
257
        self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
258
            0, QtWidgets.QHeaderView.Stretch
259
        )
260
        self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
261
            1, QtWidgets.QHeaderView.ResizeToContents
262
        )
263
        self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
264
            2, QtWidgets.QHeaderView.ResizeToContents
265
        )
266
        self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
267
            3, QtWidgets.QHeaderView.ResizeToContents
268
        )
269

270
        if update_counter > 0:
271
            self.dlg.buttonUpdateAll.setEnabled(True)
272
        else:
273
            self.dlg.buttonUpdateAll.setEnabled(False)
274

275
    def _get_dependent_addons(self, package):
276
        dependent_addons = []
277
        for addon in self.addons:
278
            # if addon.installed_version is not None:
279
            if package.lower() in addon.python_requires:
280
                dependent_addons.append({"name": addon.name, "optional": False})
281
            elif package.lower() in addon.python_optional:
282
                dependent_addons.append({"name": addon.name, "optional": True})
283
        return dependent_addons
284

285
    def _parse_pip_list_output(self, all_packages, outdated_packages) -> Dict[str, Dict[str, str]]:
286
        """Parses the output from pip into a dictionary with update information in it. The pip
287
        output should be an array of lines of text."""
288

289
        # All Packages output looks like this:
290
        # Package    Version
291
        # ---------- -------
292
        # gitdb      4.0.9
293
        # setuptools 41.2.0
294

295
        # Outdated Packages output looks like this:
296
        # Package    Version Latest Type
297
        # ---------- ------- ------ -----
298
        # pip        21.0.1  22.1.2 wheel
299
        # setuptools 41.2.0  63.2.0 wheel
300

301
        packages = {}
302
        skip_counter = 0
303
        for line in all_packages:
304
            if skip_counter < 2:
305
                skip_counter += 1
306
                continue
307
            entries = line.split()
308
            if len(entries) > 1:
309
                package_name = entries[0]
310
                installed_version = entries[1]
311
                packages[package_name] = {
312
                    "installed_version": installed_version,
313
                    "available_version": "",
314
                }
315

316
        skip_counter = 0
317
        for line in outdated_packages:
318
            if skip_counter < 2:
319
                skip_counter += 1
320
                continue
321
            entries = line.split()
322
            if len(entries) > 1:
323
                package_name = entries[0]
324
                installed_version = entries[1]
325
                available_version = entries[2]
326
                packages[package_name] = {
327
                    "installed_version": installed_version,
328
                    "available_version": available_version,
329
                }
330

331
        return packages
332

333
    def _update_package(self, package_name) -> None:
334
        """Run pip --upgrade on the given package. Updates all dependent packages as well."""
335
        for line in range(self.dlg.tableWidget.rowCount()):
336
            if self.dlg.tableWidget.item(line, 0).text() == package_name:
337
                self.dlg.tableWidget.setItem(
338
                    line,
339
                    2,
340
                    QtWidgets.QTableWidgetItem(translate("AddonsInstaller", "Updating...")),
341
                )
342
                break
343
        QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
344

345
        try:
346
            FreeCAD.Console.PrintLog(
347
                f"Running 'pip install --upgrade --target {self.vendor_path} {package_name}'\n"
348
            )
349
            call_pip(["install", "--upgrade", package_name, "--target", self.vendor_path])
350
            self._create_list_from_pip()
351
            while self.worker_thread.isRunning():
352
                QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
353
        except PipFailed as e:
354
            FreeCAD.Console.PrintError(str(e) + "\n")
355
            return
356
        QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
357

358
    def _update_all_packages(self, package_list) -> None:
359
        """Updates all packages with available updates."""
360
        updates = []
361
        for package_name, package_details in package_list.items():
362
            if (
363
                len(package_details["available_version"]) > 0
364
                and package_details["available_version"] != package_details["installed_version"]
365
            ):
366
                updates.append(package_name)
367

368
        FreeCAD.Console.PrintLog(f"Running update for {len(updates)} Python packages...\n")
369
        for package_name in updates:
370
            self._update_package(package_name)
371

372
    @classmethod
373
    def migrate_old_am_installations(cls) -> bool:
374
        """Move packages installed before the Addon Manager switched to a versioned directory
375
        structure into the versioned structure. Returns True if a migration was done, or false
376
        if no migration was needed."""
377

378
        migrated = False
379

380
        old_directory = os.path.join(FreeCAD.getUserAppDataDir(), "AdditionalPythonPackages")
381

382
        new_directory = utils.get_pip_target_directory()
383
        new_directory_name = new_directory.rsplit(os.path.sep, 1)[1]
384

385
        if not os.path.exists(old_directory) or os.path.exists(
386
            os.path.join(old_directory, "MIGRATION_COMPLETE")
387
        ):
388
            # Nothing to migrate
389
            return False
390

391
        os.makedirs(new_directory, mode=0o777, exist_ok=True)
392

393
        for content_item in os.listdir(old_directory):
394
            if content_item == new_directory_name:
395
                continue
396
            old_path = os.path.join(old_directory, content_item)
397
            new_path = os.path.join(new_directory, content_item)
398
            FreeCAD.Console.PrintLog(
399
                f"Moving {content_item} into the new (versioned) directory structure\n"
400
            )
401
            FreeCAD.Console.PrintLog(f"   {old_path} --> {new_path}\n")
402
            shutil.move(old_path, new_path)
403
            migrated = True
404

405
        sys.path.append(new_directory)
406
        cls._add_current_python_version()
407

408
        with open(os.path.join(old_directory, "MIGRATION_COMPLETE"), "w", encoding="utf-8") as f:
409
            f.write("Files originally installed in this directory have been migrated to:\n")
410
            f.write(new_directory)
411
            f.write(
412
                "\nThe existence of this file prevents the Addon Manager from "
413
                "attempting the migration again.\n"
414
            )
415
        return migrated
416

417
    @classmethod
418
    def get_known_python_versions(cls) -> List[Tuple[int, int, int]]:
419
        """Get the list of Python versions that the Addon Manager has seen before."""
420
        pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
421
        known_python_versions_string = pref.GetString("KnownPythonVersions", "[]")
422
        known_python_versions = json.loads(known_python_versions_string)
423
        return known_python_versions
424

425
    @classmethod
426
    def _add_current_python_version(cls) -> None:
427
        known_python_versions = cls.get_known_python_versions()
428
        major, minor, _ = platform.python_version_tuple()
429
        if not [major, minor] in known_python_versions:
430
            known_python_versions.append((major, minor))
431
        pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
432
        pref.SetString("KnownPythonVersions", json.dumps(known_python_versions))
433

434
    @classmethod
435
    def _current_python_version_is_new(cls) -> bool:
436
        """Returns True if this is the first time the Addon Manager has seen this version of
437
        Python"""
438
        known_python_versions = cls.get_known_python_versions()
439
        major, minor, _ = platform.python_version_tuple()
440
        if not [major, minor] in known_python_versions:
441
            return True
442
        return False
443

444
    def _load_old_package_list(self) -> List[str]:
445
        """Gets the list of packages from the package installation manifest"""
446

447
        known_python_versions = self.get_known_python_versions()
448
        if not known_python_versions:
449
            return []
450
        last_version = known_python_versions[-1]
451
        expected_directory = f"py{last_version[0]}{last_version[1]}"
452
        expected_directory = os.path.join(
453
            FreeCAD.getUserAppDataDir(), "AdditionalPythonPackages", expected_directory
454
        )
455
        # For now just do this synchronously
456
        worker_object = PythonPackageManager.PipRunner(expected_directory)
457
        worker_object.process()
458
        packages = self._parse_pip_list_output(
459
            worker_object.all_packages_stdout, worker_object.outdated_packages_stdout
460
        )
461
        return packages.keys()
462

463
    def _reinstall_all_packages(self) -> None:
464
        """Loads the package manifest from another Python version, and installs the same packages
465
        for the current (presumably new) version of Python."""
466

467
        packages = self._load_old_package_list()
468
        args = ["install"]
469
        args.extend(packages)
470
        args.extend(["--target", self.vendor_path])
471

472
        try:
473
            call_pip(args)
474
        except PipFailed as e:
475
            FreeCAD.Console.PrintError(str(e) + "\n")
476
            return
477