onnxruntime

1
# Copyright (c) Microsoft Corporation. All rights reserved.
2
# Licensed under the MIT License.
3

4
import contextlib
5
import logging
6
import os
7
import platform
8
import re
9
import shutil
10
import stat
11
import subprocess
12
import tempfile
13
import urllib.parse
14
import urllib.request
15

16
AZCOPY_VERSION = "10.4.3"
17

18
# See here for instructions on getting stable download links:
19
# https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10#obtain-a-static-download-link
20
_AZCOPY_DOWNLOAD_URLS = {
21
    "Linux": "https://azcopyvnext.azureedge.net/release20200501/azcopy_linux_amd64_10.4.3.tar.gz",
22
    "Darwin": "https://azcopyvnext.azureedge.net/release20200501/azcopy_darwin_amd64_10.4.3.zip",
23
    "Windows": "https://azcopyvnext.azureedge.net/release20200501/azcopy_windows_amd64_10.4.3.zip",
24
}
25

26
_log = logging.getLogger("util.get_azcopy")
27

28

29
def _check_version(azcopy_path):
30
    proc = subprocess.run([azcopy_path, "--version"], stdout=subprocess.PIPE, text=True)  # noqa: PLW1510
31
    match = re.search(r"\d+(?:\.\d+)+", proc.stdout)
32

33
    if not match:
34
        raise RuntimeError("Failed to determine azcopy version.")
35

36
    return match.group(0) == AZCOPY_VERSION
37

38

39
def _find_azcopy(start_dir):
40
    for root, _, file_names in os.walk(start_dir):
41
        for file_name in file_names:
42
            if file_name == "azcopy" or file_name == "azcopy.exe":
43
                return os.path.join(root, file_name)
44
    raise RuntimeError(f"Failed to azcopy in '{start_dir}'.")
45

46

47
@contextlib.contextmanager
48
def get_azcopy(local_azcopy_path="azcopy"):
49
    """
50
    Creates a context manager that returns a path to a particular version of
51
    azcopy (specified in AZCOPY_VERSION). Downloads a temporary copy if needed.
52

53
    :param local_azcopy_path: Path to a local azcopy to try first.
54

55
    Example usage:
56
        with get_azcopy() as azcopy_path:
57
            subprocess.run([azcopy_path, "--version"])
58
    """
59
    with contextlib.ExitStack() as context_stack:
60
        azcopy_path = shutil.which(local_azcopy_path)
61

62
        if azcopy_path is None or not _check_version(azcopy_path):
63
            temp_dir = context_stack.enter_context(tempfile.TemporaryDirectory())
64

65
            download_url = _AZCOPY_DOWNLOAD_URLS[platform.system()]
66
            download_basename = urllib.parse.urlsplit(download_url).path.rsplit("/", 1)[-1]
67
            assert len(download_basename) > 0
68
            downloaded_path = os.path.join(temp_dir, download_basename)
69

70
            _log.info(f"Downloading azcopy from '{download_url}'...")
71
            urllib.request.urlretrieve(download_url, downloaded_path)
72

73
            extracted_path = os.path.join(temp_dir, "azcopy")
74
            shutil.unpack_archive(downloaded_path, extracted_path)
75

76
            azcopy_path = _find_azcopy(extracted_path)
77

78
            os.chmod(azcopy_path, stat.S_IXUSR)
79

80
        yield azcopy_path
81