intel-extension-for-pytorch

1
# Referenced from https://github.com/pytorch/pytorch/blob/master/torch/utils/collect_env.py
2
# Run it with `python collect_env.py`.
3
import locale
4
import re
5
import subprocess
6
import sys
7
import os
8
from collections import namedtuple
9

10
try:
11
    import torch
12

13
    TORCH_AVAILABLE = True
14
except (ImportError, NameError, AttributeError, OSError):
15
    TORCH_AVAILABLE = False
16

17
try:
18
    import intel_extension_for_pytorch as ipex
19

20
    IPEX_AVAILABLE = True
21
except (ImportError, NameError, AttributeError, OSError):
22
    IPEX_AVAILABLE = False
23

24

25
# System Environment Information
26
SystemEnv = namedtuple(
27
    "SystemEnv",
28
    [
29
        "torch_version",
30
        "torch_cxx11_abi",
31
        "ipex_version",
32
        "ipex_gitrev",
33
        "build_type",
34
        "gcc_version",
35
        "clang_version",
36
        "icx_version",
37
        "cmake_version",
38
        "os",
39
        "libc_version",
40
        "python_version",
41
        "python_platform",
42
        "is_xpu_available",
43
        "dpcpp_runtime_version",
44
        "mkl_version",
45
        "gpu_models",
46
        "intel_opencl_version",
47
        "level_zero_version",
48
        "pip_version",  # 'pip' or 'pip3'
49
        "pip_packages",
50
        "conda_packages",
51
        "cpu_info",
52
    ],
53
)
54

55

56
def run(command):
57
    """Returns (return-code, stdout, stderr)"""
58
    my_env = os.environ.copy()
59
    my_env["LC_ALL"] = "C"
60
    p = subprocess.Popen(
61
        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env, shell=True
62
    )
63
    raw_output, raw_err = p.communicate()
64
    rc = p.returncode
65
    if get_platform() == "win32":
66
        enc = "oem"
67
    else:
68
        enc = locale.getpreferredencoding()
69
    output = raw_output.decode(enc)
70
    err = raw_err.decode(enc)
71
    return rc, output.strip(), err.strip()
72

73

74
def run_and_read_all(run_lambda, command):
75
    """Runs command using run_lambda; reads and returns entire output if rc is 0"""
76
    rc, out, _ = run_lambda(command)
77
    if rc != 0:
78
        return None
79
    return out
80

81

82
def run_and_parse_first_match(run_lambda, command, regex):
83
    """Runs command using run_lambda, returns the first regex match if it exists"""
84
    rc, out, _ = run_lambda(command)
85
    if rc != 0:
86
        return None
87
    match = re.search(regex, out)
88
    if match is None:
89
        return None
90
    return match.group(1)
91

92

93
def run_and_return_first_line(run_lambda, command):
94
    """Runs command using run_lambda and returns first line if output is not empty"""
95
    rc, out, _ = run_lambda(command)
96
    if rc != 0:
97
        return None
98
    return out.split("\n")[0]
99

100

101
def get_conda_packages(run_lambda):
102
    conda = os.environ.get("CONDA_EXE", "conda")
103
    out = run_and_read_all(run_lambda, "{} list".format(conda))
104
    if out is None:
105
        return out
106

107
    return "\n".join(
108
        line
109
        for line in out.splitlines()
110
        if not line.startswith("#")
111
        and any(
112
            name in line
113
            for name in {
114
                "torch",
115
                "numpy",
116
                "mkl",
117
            }
118
        )
119
    )
120

121

122
def get_gcc_version(run_lambda):
123
    return run_and_parse_first_match(run_lambda, "gcc --version", r"gcc (.*)")
124

125

126
def get_clang_version(run_lambda):
127
    return run_and_parse_first_match(
128
        run_lambda, "clang --version", r"clang version (.*)"
129
    )
130

131

132
def get_icx_version(run_lambda):
133
    return run_and_parse_first_match(
134
        run_lambda, "icx --version", r"Intel\(R\) oneAPI DPC\+\+\/C\+\+ Compiler (.*)"
135
    )
136

137

138
def get_cmake_version(run_lambda):
139
    return run_and_parse_first_match(run_lambda, "cmake --version", r"cmake (.*)")
140

141

142
def get_pkg_version(run_lambda, pkg):
143
    txt = ""
144
    index = -1
145
    if get_platform() == "linux":
146
        mgr_name = ""
147
        if mgr_name == "":
148
            rc, _, _ = run("which dpkg")
149
            if rc == 0:
150
                mgr_name = "dpkg"
151
        if mgr_name == "":
152
            rc, _, _ = run("which yum")
153
            if rc == 0:
154
                mgr_name = "yum"
155
        if mgr_name == "":
156
            rc, _, _ = run("which dnf")
157
            if rc == 0:
158
                mgr_name = "dnf"
159
        if mgr_name != "":
160
            cmd = ""
161
            if mgr_name == "yum" or mgr_name == "dnf":
162
                index = 1
163
                pkg_name = ""
164
                if pkg == "intel_opencl":
165
                    pkg_name = "intel-opencl"
166
                if pkg == "level_zero":
167
                    pkg_name = "intel-level-zero-gpu"
168
                if pkg_name != "":
169
                    cmd = f"{mgr_name} list | grep {pkg_name}"
170
            if mgr_name == "dpkg":
171
                index = 2
172
                pkg_name = ""
173
                if pkg == "intel_opencl":
174
                    pkg_name = "intel-opencl-icd"
175
                if pkg == "level_zero":
176
                    pkg_name = "intel-level-zero-gpu"
177
                if pkg_name != "":
178
                    cmd = f"{mgr_name} -l | grep {pkg_name}"
179
            if cmd != "":
180
                txt = run_and_read_all(run_lambda, cmd)
181
    lst_txt = []
182
    if txt:
183
        lst_txt = re.sub(" +", " ", txt).split(" ")
184
    if len(lst_txt) > index and index != -1:
185
        txt = lst_txt[index]
186
    else:
187
        txt = "N/A"
188
    return txt
189

190

191
def get_gpu_info(run_lambda):
192
    if TORCH_AVAILABLE and IPEX_AVAILABLE:
193
        devices = [
194
            f"[{i}] {torch.xpu.get_device_properties(i)}"
195
            for i in range(torch.xpu.device_count())
196
        ]
197
        return "\n".join(devices)
198
    else:
199
        return "N/A"
200

201

202
def get_running_dpcpp_version(run_lambda):
203
    return run_and_read_all(
204
        run_lambda, 'env | grep CMPLR_ROOT | rev | cut -d "/" -f 1 | rev'
205
    )
206

207

208
def get_mkl_version(run_lambda):
209
    return run_and_read_all(
210
        run_lambda, 'env | grep MKLROOT | rev | cut -d "/" -f 1 | rev'
211
    )
212

213

214
def get_cpu_info(run_lambda):
215
    rc, out, err = 0, "", ""
216
    if get_platform() == "linux":
217
        rc, out, err = run_lambda("lscpu")
218
    elif get_platform() == "win32":
219
        rc, out, err = run_lambda(
220
            "wmic cpu get Name,Manufacturer,Family,Architecture,ProcessorType,DeviceID,\
221
        CurrentClockSpeed,MaxClockSpeed,L2CacheSize,L2CacheSpeed,Revision /VALUE"
222
        )
223
    elif get_platform() == "darwin":
224
        rc, out, err = run_lambda("sysctl -n machdep.cpu.brand_string")
225
    cpu_info = "N/A"
226
    if rc == 0:
227
        cpu_info = out
228
    else:
229
        cpu_info = err
230
    return cpu_info
231

232

233
def get_platform():
234
    if sys.platform.startswith("linux"):
235
        return "linux"
236
    elif sys.platform.startswith("win32"):
237
        return "win32"
238
    elif sys.platform.startswith("cygwin"):
239
        return "cygwin"
240
    elif sys.platform.startswith("darwin"):
241
        return "darwin"
242
    else:
243
        return sys.platform
244

245

246
def get_mac_version(run_lambda):
247
    return run_and_parse_first_match(run_lambda, "sw_vers -productVersion", r"(.*)")
248

249

250
def get_windows_version(run_lambda):
251
    system_root = os.environ.get("SYSTEMROOT", "C:\\Windows")
252
    wmic_cmd = os.path.join(system_root, "System32", "Wbem", "wmic")
253
    findstr_cmd = os.path.join(system_root, "System32", "findstr")
254
    return run_and_read_all(
255
        run_lambda, "{} os get Caption | {} /v Caption".format(wmic_cmd, findstr_cmd)
256
    )
257

258

259
def get_lsb_version(run_lambda):
260
    return run_and_parse_first_match(
261
        run_lambda, "lsb_release -a", r"Description:\t(.*)"
262
    )
263

264

265
def check_release_file(run_lambda):
266
    return run_and_parse_first_match(
267
        run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"'
268
    )
269

270

271
def get_os(run_lambda):
272
    from platform import machine
273

274
    platform = get_platform()
275

276
    if platform == "win32" or platform == "cygwin":
277
        return get_windows_version(run_lambda)
278

279
    if platform == "darwin":
280
        version = get_mac_version(run_lambda)
281
        if version is None:
282
            return None
283
        return "macOS {} ({})".format(version, machine())
284

285
    if platform == "linux":
286
        # Ubuntu/Debian based
287
        desc = get_lsb_version(run_lambda)
288
        if desc is not None:
289
            return "{} ({})".format(desc, machine())
290

291
        # Try reading /etc/*-release
292
        desc = check_release_file(run_lambda)
293
        if desc is not None:
294
            return "{} ({})".format(desc, machine())
295

296
        return "{} ({})".format(platform, machine())
297

298
    # Unknown platform
299
    return platform
300

301

302
def get_python_platform():
303
    import platform
304

305
    return platform.platform()
306

307

308
def get_libc_version():
309
    import platform
310

311
    if get_platform() != "linux":
312
        return "N/A"
313
    return "-".join(platform.libc_ver())
314

315

316
def get_pip_packages(run_lambda):
317
    """Returns `pip list` output. Note: will also find conda-installed pytorch
318
    and numpy packages."""
319

320
    # People generally have `pip` as `pip` or `pip3`
321
    # But here it is incoved as `python -mpip`
322
    def run_with_pip(pip):
323
        out = run_and_read_all(run_lambda, "{} list --format=freeze".format(pip))
324
        return "\n".join(
325
            line
326
            for line in out.splitlines()
327
            if any(
328
                name in line
329
                for name in {
330
                    "torch",
331
                    "numpy",
332
                    "mypy",
333
                }
334
            )
335
        )
336

337
    pip_version = "pip3" if sys.version[0] == "3" else "pip"
338
    out = run_with_pip(sys.executable + " -mpip")
339

340
    return pip_version, out
341

342

343
def get_env_info():
344
    run_lambda = run
345
    pip_version, pip_list_output = get_pip_packages(run_lambda)
346

347
    if TORCH_AVAILABLE:
348
        torch_version_str = torch.__version__
349
        torch_cxx11_abi_str = torch._C._GLIBCXX_USE_CXX11_ABI
350
    else:
351
        torch_version_str = torch_cxx11_abi_str = "N/A"
352

353
    if IPEX_AVAILABLE:
354
        ipex_version_str = ipex.__version__
355
        try:
356
            import intel_extension_for_pytorch._version as ver
357
        except ImportError:
358
            import intel_extension_for_pytorch.version as ver
359
        try:
360
            ipex_gitrev_str = ver.__ipex_gitrev__
361
        except AttributeError:
362
            ipex_gitrev_str = ver.__gitrev__
363
        try:
364
            build_type_str = str(ver.__build_type__)
365
        except AttributeError:
366
            build_type_str = str(ver.__mode__)
367
        try:
368
            xpu_available_str = str(torch.xpu.is_available())
369
        except AttributeError:
370
            xpu_available_str = False
371
    else:
372
        ipex_version_str = ipex_gitrev_str = "N/A"
373
        build_type_str = xpu_available_str = "N/A"
374

375
    sys_version = sys.version.replace("\n", " ")
376

377
    return SystemEnv(
378
        torch_version=torch_version_str,
379
        torch_cxx11_abi=torch_cxx11_abi_str,
380
        ipex_version=ipex_version_str,
381
        ipex_gitrev=ipex_gitrev_str,
382
        build_type=build_type_str,
383
        python_version="{} ({}-bit runtime)".format(
384
            sys_version, sys.maxsize.bit_length() + 1
385
        ),
386
        python_platform=get_python_platform(),
387
        is_xpu_available=xpu_available_str,
388
        dpcpp_runtime_version=get_running_dpcpp_version(run_lambda),
389
        mkl_version=get_mkl_version(run_lambda),
390
        gpu_models=f"\n{get_gpu_info(run_lambda)}",
391
        intel_opencl_version=get_pkg_version(run_lambda, "intel_opencl"),
392
        level_zero_version=get_pkg_version(run_lambda, "level_zero"),
393
        pip_version=pip_version,
394
        pip_packages=pip_list_output,
395
        conda_packages=get_conda_packages(run_lambda),
396
        os=get_os(run_lambda),
397
        libc_version=get_libc_version(),
398
        gcc_version=get_gcc_version(run_lambda),
399
        clang_version=get_clang_version(run_lambda),
400
        icx_version=get_icx_version(run_lambda),
401
        cmake_version=get_cmake_version(run_lambda),
402
        cpu_info=get_cpu_info(run_lambda),
403
    )
404

405

406
env_info_fmt = """
407
PyTorch version: {torch_version}
408
PyTorch CXX11 ABI: {torch_cxx11_abi}
409
IPEX version: {ipex_version}
410
IPEX commit: {ipex_gitrev}
411
Build type: {build_type}
412

413
OS: {os}
414
GCC version: {gcc_version}
415
Clang version: {clang_version}
416
IGC version: {icx_version}
417
CMake version: {cmake_version}
418
Libc version: {libc_version}
419

420
Python version: {python_version}
421
Python platform: {python_platform}
422
Is XPU available: {is_xpu_available}
423
DPCPP runtime version: {dpcpp_runtime_version}
424
MKL version: {mkl_version}
425
GPU models and configuration: {gpu_models}
426
Intel OpenCL ICD version: {intel_opencl_version}
427
Level Zero version: {level_zero_version}
428

429
CPU:
430
{cpu_info}
431

432
Versions of relevant libraries:
433
{pip_packages}
434
{conda_packages}
435
""".strip()
436

437

438
def pretty_str(envinfo):
439
    def replace_nones(dct, replacement="Could not collect"):
440
        for key in dct.keys():
441
            if dct[key] is not None:
442
                continue
443
            dct[key] = replacement
444
        return dct
445

446
    def replace_empties(dct, replacement="Could not collect"):
447
        for key in dct.keys():
448
            if dct[key] is not None and len(dct[key]) > 0:
449
                continue
450
            dct[key] = replacement
451
        return dct
452

453
    def replace_bools(dct, true="Yes", false="No"):
454
        for key in dct.keys():
455
            if dct[key] is True:
456
                dct[key] = true
457
            elif dct[key] is False:
458
                dct[key] = false
459
        return dct
460

461
    def prepend(text, tag="[prepend]"):
462
        lines = text.split("\n")
463
        updated_lines = [tag + line for line in lines]
464
        return "\n".join(updated_lines)
465

466
    def replace_if_empty(text, replacement="No relevant packages"):
467
        if text is not None and len(text) == 0:
468
            return replacement
469
        return text
470

471
    def maybe_start_on_next_line(string):
472
        # If `string` is multiline, prepend a \n to it.
473
        if string is not None and len(string.split("\n")) > 1:
474
            return "\n{}\n".format(string)
475
        return string
476

477
    mutable_dict = envinfo._asdict()
478

479
    # Replace True with Yes, False with No
480
    mutable_dict = replace_bools(mutable_dict)
481

482
    # Replace all None objects with 'N/A'
483
    mutable_dict = replace_nones(mutable_dict, replacement="N/A")
484

485
    # Replace all empty objects with 'N/A'
486
    mutable_dict = replace_empties(mutable_dict, replacement="N/A")
487

488
    # If either of these are '', replace with 'No relevant packages'
489
    mutable_dict["pip_packages"] = replace_if_empty(mutable_dict["pip_packages"])
490
    mutable_dict["conda_packages"] = replace_if_empty(mutable_dict["conda_packages"])
491

492
    # Tag conda and pip packages with a prefix
493
    # If they were previously None, they'll show up as ie '[conda] Could not collect'
494
    if mutable_dict["pip_packages"]:
495
        mutable_dict["pip_packages"] = prepend(
496
            mutable_dict["pip_packages"], "[{}] ".format(envinfo.pip_version)
497
        )
498
    if mutable_dict["conda_packages"]:
499
        mutable_dict["conda_packages"] = prepend(
500
            mutable_dict["conda_packages"], "[conda] "
501
        )
502
    mutable_dict["cpu_info"] = envinfo.cpu_info
503
    return env_info_fmt.format(**mutable_dict)
504

505

506
def get_pretty_env_info():
507
    return pretty_str(get_env_info())
508

509

510
def main():
511
    print("Collecting environment information...")
512
    output = get_pretty_env_info()
513
    print(output)
514

515

516
if __name__ == "__main__":
517
    main()
518