pytorch

1
# Documentation: https://docs.microsoft.com/en-us/rest/api/azure/devops/build/?view=azure-devops-rest-6.0
2

3
import json
4
import os
5
import re
6
import sys
7
import time
8

9
import requests
10

11
AZURE_PIPELINE_BASE_URL = "https://aiinfra.visualstudio.com/PyTorch/"
12
AZURE_DEVOPS_PAT_BASE64 = os.environ.get("AZURE_DEVOPS_PAT_BASE64_SECRET", "")
13
PIPELINE_ID = "911"
14
PROJECT_ID = "0628bce4-2d33-499e-bac5-530e12db160f"
15
TARGET_BRANCH = os.environ.get("CIRCLE_BRANCH", "main")
16
TARGET_COMMIT = os.environ.get("CIRCLE_SHA1", "")
17

18
build_base_url = AZURE_PIPELINE_BASE_URL + "_apis/build/builds?api-version=6.0"
19

20
s = requests.Session()
21
s.headers.update({"Authorization": "Basic " + AZURE_DEVOPS_PAT_BASE64})
22

23

24
def submit_build(pipeline_id, project_id, source_branch, source_version):
25
    print("Submitting build for branch: " + source_branch)
26
    print("Commit SHA1: ", source_version)
27

28
    run_build_raw = s.post(
29
        build_base_url,
30
        json={
31
            "definition": {"id": pipeline_id},
32
            "project": {"id": project_id},
33
            "sourceBranch": source_branch,
34
            "sourceVersion": source_version,
35
        },
36
    )
37

38
    try:
39
        run_build_json = run_build_raw.json()
40
    except json.decoder.JSONDecodeError as e:
41
        print(e)
42
        print(
43
            "Failed to parse the response. Check if the Azure DevOps PAT is incorrect or expired."
44
        )
45
        sys.exit(-1)
46

47
    build_id = run_build_json["id"]
48

49
    print("Submitted bulid: " + str(build_id))
50
    print("Bulid URL: " + run_build_json["url"])
51
    return build_id
52

53

54
def get_build(_id):
55
    get_build_url = (
56
        AZURE_PIPELINE_BASE_URL + f"/_apis/build/builds/{_id}?api-version=6.0"
57
    )
58
    get_build_raw = s.get(get_build_url)
59
    return get_build_raw.json()
60

61

62
def get_build_logs(_id):
63
    get_build_logs_url = (
64
        AZURE_PIPELINE_BASE_URL + f"/_apis/build/builds/{_id}/logs?api-version=6.0"
65
    )
66
    get_build_logs_raw = s.get(get_build_logs_url)
67
    return get_build_logs_raw.json()
68

69

70
def get_log_content(url):
71
    resp = s.get(url)
72
    return resp.text
73

74

75
def wait_for_build(_id):
76
    build_detail = get_build(_id)
77
    build_status = build_detail["status"]
78

79
    while build_status == "notStarted":
80
        print("Waiting for run to start: " + str(_id))
81
        sys.stdout.flush()
82
        try:
83
            build_detail = get_build(_id)
84
            build_status = build_detail["status"]
85
        except Exception as e:
86
            print("Error getting build")
87
            print(e)
88

89
        time.sleep(30)
90

91
    print("Bulid started: ", str(_id))
92

93
    handled_logs = set()
94
    while build_status == "inProgress":
95
        try:
96
            print("Waiting for log: " + str(_id))
97
            logs = get_build_logs(_id)
98
        except Exception as e:
99
            print("Error fetching logs")
100
            print(e)
101
            time.sleep(30)
102
            continue
103

104
        for log in logs["value"]:
105
            log_id = log["id"]
106
            if log_id in handled_logs:
107
                continue
108
            handled_logs.add(log_id)
109
            print("Fetching log: \n" + log["url"])
110
            try:
111
                log_content = get_log_content(log["url"])
112
                print(log_content)
113
            except Exception as e:
114
                print("Error getting log content")
115
                print(e)
116
            sys.stdout.flush()
117
        build_detail = get_build(_id)
118
        build_status = build_detail["status"]
119
        time.sleep(30)
120

121
    build_result = build_detail["result"]
122

123
    print("Bulid status: " + build_status)
124
    print("Bulid result: " + build_result)
125

126
    return build_status, build_result
127

128

129
if __name__ == "__main__":
130
    # Convert the branch name for Azure DevOps
131
    match = re.search(r"pull/(\d+)", TARGET_BRANCH)
132
    if match is not None:
133
        pr_num = match.group(1)
134
        SOURCE_BRANCH = f"refs/pull/{pr_num}/head"
135
    else:
136
        SOURCE_BRANCH = f"refs/heads/{TARGET_BRANCH}"
137

138
    MAX_RETRY = 2
139
    retry = MAX_RETRY
140

141
    while retry > 0:
142
        build_id = submit_build(PIPELINE_ID, PROJECT_ID, SOURCE_BRANCH, TARGET_COMMIT)
143
        build_status, build_result = wait_for_build(build_id)
144

145
        if build_result != "succeeded":
146
            retry = retry - 1
147
            if retry > 0:
148
                print("Retrying... remaining attempt: " + str(retry))
149
                # Wait a bit before retrying
150
                time.sleep((MAX_RETRY - retry) * 120)
151
                continue
152
            else:
153
                print("No more chance to retry. Giving up.")
154
                sys.exit(-1)
155
        else:
156
            break
157