numpy

1
#!/usr/bin/env python3
2
"""
3
Script to download NumPy wheels from the Anaconda staging area.
4

5
Usage::
6

7
    $ ./tools/download-wheels.py <version> -w <optional-wheelhouse>
8

9
The default wheelhouse is ``release/installers``.
10

11
Dependencies
12
------------
13

14
- beautifulsoup4
15
- urllib3
16

17
Examples
18
--------
19

20
While in the repository root::
21

22
    $ python tools/download-wheels.py 1.19.0
23
    $ python tools/download-wheels.py 1.19.0 -w ~/wheelhouse
24

25
"""
26
import os
27
import re
28
import shutil
29
import argparse
30

31
import urllib3
32
from bs4 import BeautifulSoup
33

34
__version__ = "0.1"
35

36
# Edit these for other projects.
37
STAGING_URL = "https://anaconda.org/multibuild-wheels-staging/numpy"
38
PREFIX = "numpy"
39

40
# Name endings of the files to download.
41
WHL = r"-.*\.whl$"
42
ZIP = r"\.zip$"
43
GZIP = r"\.tar\.gz$"
44
SUFFIX = rf"({WHL}|{GZIP}|{ZIP})"
45

46

47
def get_wheel_names(version):
48
    """ Get wheel names from Anaconda HTML directory.
49

50
    This looks in the Anaconda multibuild-wheels-staging page and
51
    parses the HTML to get all the wheel names for a release version.
52

53
    Parameters
54
    ----------
55
    version : str
56
        The release version. For instance, "1.18.3".
57

58
    """
59
    ret = []
60
    http = urllib3.PoolManager(cert_reqs="CERT_REQUIRED")
61
    tmpl = re.compile(rf"^.*{PREFIX}-{version}{SUFFIX}")
62
    # TODO: generalize this by searching for `showing 1 of N` and
63
    # looping over N pages, starting from 1
64
    for i in range(1, 3):
65
        index_url = f"{STAGING_URL}/files?page={i}"
66
        index_html = http.request("GET", index_url)
67
        soup = BeautifulSoup(index_html.data, "html.parser")
68
        ret += soup.find_all(string=tmpl)
69
    return ret
70

71

72
def download_wheels(version, wheelhouse, test=False):
73
    """Download release wheels.
74

75
    The release wheels for the given NumPy version are downloaded
76
    into the given directory.
77

78
    Parameters
79
    ----------
80
    version : str
81
        The release version. For instance, "1.18.3".
82
    wheelhouse : str
83
        Directory in which to download the wheels.
84

85
    """
86
    http = urllib3.PoolManager(cert_reqs="CERT_REQUIRED")
87
    wheel_names = get_wheel_names(version)
88

89
    for i, wheel_name in enumerate(wheel_names):
90
        wheel_url = f"{STAGING_URL}/{version}/download/{wheel_name}"
91
        wheel_path = os.path.join(wheelhouse, wheel_name)
92
        with open(wheel_path, "wb") as f:
93
            with http.request("GET", wheel_url, preload_content=False,) as r:
94
                info = r.info()
95
                length = int(info.get('Content-Length', '0'))
96
                if length == 0:
97
                    length = 'unknown size'
98
                else:
99
                    length = f"{(length / 1024 / 1024):.2f}MB"
100
                print(f"{i + 1:<4}{wheel_name} {length}")
101
                if not test:
102
                    shutil.copyfileobj(r, f)
103
    print(f"\nTotal files downloaded: {len(wheel_names)}")
104

105

106
if __name__ == "__main__":
107
    parser = argparse.ArgumentParser()
108
    parser.add_argument(
109
        "version",
110
        help="NumPy version to download.")
111
    parser.add_argument(
112
        "-w", "--wheelhouse",
113
        default=os.path.join(os.getcwd(), "release", "installers"),
114
        help="Directory in which to store downloaded wheels\n"
115
             "[defaults to <cwd>/release/installers]")
116
    parser.add_argument(
117
        "-t", "--test",
118
        action = 'store_true',
119
        help="only list available wheels, do not download")
120

121
    args = parser.parse_args()
122

123
    wheelhouse = os.path.expanduser(args.wheelhouse)
124
    if not os.path.isdir(wheelhouse):
125
        raise RuntimeError(
126
            f"{wheelhouse} wheelhouse directory is not present."
127
            " Perhaps you need to use the '-w' flag to specify one.")
128

129
    download_wheels(args.version, wheelhouse, test=args.test)
130