llvm-project

1
#!/usr/bin/env bash
2
#===----------------------------------------------------------------------===##
3
#
4
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5
# See https://llvm.org/LICENSE.txt for license information.
6
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7
#
8
#===----------------------------------------------------------------------===##
9

10
#
11
# This file generates a Buildkite pipeline that triggers the various CI jobs for
12
# the LLVM project during pre-commit CI.
13
#
14
# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
15
#
16
# As this outputs a yaml file, it's possible to log messages to stderr or
17
# prefix with "#".
18

19

20
set -eu
21
set -o pipefail
22

23
# Environment variables script works with:
24

25
# Set by buildkite
26
: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
27
: ${BUILDKITE_COMMIT:=}
28
: ${BUILDKITE_BRANCH:=}
29
# Fetch origin to have an up to date merge base for the diff.
30
git fetch origin
31
# List of files affected by this commit
32
: ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
33
# Filter rules for generic windows tests
34
: ${WINDOWS_AGENTS:='{"queue": "windows"}'}
35
# Filter rules for generic linux tests
36
: ${LINUX_AGENTS:='{"queue": "linux"}'}
37

38
reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
39
if [[ "${reviewID}" != "" ]]; then
40
  buildMessage="https://llvm.org/${reviewID}"
41
else
42
  buildMessage="Push to branch ${BUILDKITE_BRANCH}"
43
fi
44

45
cat <<EOF
46
steps:
47
EOF
48

49
echo "Files modified:" >&2
50
echo "$MODIFIED_FILES" >&2
51
modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
52
echo "Directories modified:" >&2
53
echo "$modified_dirs" >&2
54

55
function compute-projects-to-test() {
56
  isForWindows=$1
57
  shift
58
  projects=${@}
59
  for project in ${projects}; do
60
    echo "${project}"
61
    case ${project} in
62
    lld)
63
      for p in bolt cross-project-tests; do
64
        echo $p
65
      done
66
    ;;
67
    llvm)
68
      for p in bolt clang clang-tools-extra lld lldb mlir polly; do
69
        echo $p
70
      done
71
      # Flang is not stable in Windows CI at the moment
72
      if [[ $isForWindows == 0 ]]; then
73
        echo flang
74
      fi
75
    ;;
76
    clang)
77
      # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
78
      for p in clang-tools-extra compiler-rt cross-project-tests; do
79
        echo $p
80
      done
81
    ;;
82
    clang-tools-extra)
83
      echo libc
84
    ;;
85
    mlir)
86
      # Flang is not stable in Windows CI at the moment
87
      if [[ $isForWindows == 0 ]]; then
88
        echo flang
89
      fi
90
    ;;
91
    *)
92
      # Nothing to do
93
    ;;
94
    esac
95
  done
96
}
97

98
function compute-runtimes-to-test() {
99
  projects=${@}
100
  for project in ${projects}; do
101
    case ${project} in
102
    clang)
103
      for p in libcxx libcxxabi libunwind; do
104
        echo $p
105
      done
106
    ;;
107
    *)
108
      # Nothing to do
109
    ;;
110
    esac
111
  done
112
}
113

114
function add-dependencies() {
115
  projects=${@}
116
  for project in ${projects}; do
117
    echo "${project}"
118
    case ${project} in
119
    bolt)
120
      for p in clang lld llvm; do
121
        echo $p
122
      done
123
    ;;
124
    cross-project-tests)
125
      for p in lld clang; do
126
        echo $p
127
      done
128
    ;;
129
    clang-tools-extra)
130
      for p in llvm clang; do
131
        echo $p
132
      done
133
    ;;
134
    compiler-rt|libc|openmp)
135
      echo clang lld
136
    ;;
137
    flang|lldb|libclc)
138
      for p in llvm clang; do
139
        echo $p
140
      done
141
    ;;
142
    lld|mlir|polly)
143
      echo llvm
144
    ;;
145
    *)
146
      # Nothing to do
147
    ;;
148
    esac
149
  done
150
}
151

152
function exclude-linux() {
153
  projects=${@}
154
  for project in ${projects}; do
155
    case ${project} in
156
    cross-project-tests) ;; # tests failing
157
    openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
158
    *)
159
      echo "${project}"
160
    ;;
161
    esac
162
  done
163
}
164

165
function exclude-windows() {
166
  projects=${@}
167
  for project in ${projects}; do
168
    case ${project} in
169
    cross-project-tests) ;; # tests failing
170
    compiler-rt)         ;; # tests taking too long
171
    openmp)              ;; # TODO: having trouble with the Perl installation
172
    libc)                ;; # no Windows support
173
    lldb)                ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
174
    bolt)                ;; # tests are not supported yet
175
    *)
176
      echo "${project}"
177
    ;;
178
    esac
179
  done
180
}
181

182
# Prints only projects that are both present in $modified_dirs and the passed
183
# list.
184
function keep-modified-projects() {
185
  projects=${@}
186
  for project in ${projects}; do
187
    if echo "$modified_dirs" | grep -q -E "^${project}$"; then
188
      echo "${project}"
189
    fi
190
  done
191
}
192

193
function check-targets() {
194
  projects=${@}
195
  for project in ${projects}; do
196
    case ${project} in
197
    clang-tools-extra)
198
      echo "check-clang-tools"
199
    ;;
200
    compiler-rt)
201
      echo "check-all"
202
    ;;
203
    cross-project-tests)
204
      echo "check-cross-project"
205
    ;;
206
    libcxx)
207
      echo "check-cxx"
208
    ;;
209
    libcxxabi)
210
      echo "check-cxxabi"
211
    ;;
212
    libunwind)
213
      echo "check-unwind"
214
    ;;
215
    lldb)
216
      echo "check-lldb"
217
    ;;
218
    pstl)
219
      echo "check-all"
220
    ;;
221
    libclc)
222
      echo "check-all"
223
    ;;
224
    *)
225
      echo "check-${project}"
226
    ;;
227
    esac
228
  done
229
}
230

231
# Project specific pipelines.
232

233
# If libc++ or one of the runtimes directories changed.
234
if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
235
  cat <<EOF
236
- trigger: "libcxx-ci"
237
  build:
238
    message: "${buildMessage}"
239
    commit: "${BUILDKITE_COMMIT}"
240
    branch: "${BUILDKITE_BRANCH}"
241
EOF
242
fi
243

244
# Generic pipeline for projects that have not defined custom steps.
245
#
246
# Individual projects should instead define the pre-commit CI tests that suits their
247
# needs while letting them run on the infrastructure provided by LLVM.
248

249
# Figure out which projects need to be built on each platform
250
all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
251
modified_projects="$(keep-modified-projects ${all_projects})"
252

253
linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
254
linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
255
linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
256

257
linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
258
linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
259
linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
260

261
windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
262
windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
263
windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
264

265
# Generate the appropriate pipeline
266
if [[ "${linux_projects}" != "" ]]; then
267
  cat <<EOF
268
- label: ':linux: Linux x64'
269
  artifact_paths:
270
  - 'artifacts/**/*'
271
  - '*_result.json'
272
  - 'build/test-results.xml'
273
  agents: ${LINUX_AGENTS}
274
  retry:
275
    automatic:
276
      - exit_status: -1  # Agent was lost
277
        limit: 2
278
      - exit_status: 255 # Forced agent shutdown
279
        limit: 2
280
  timeout_in_minutes: 120
281
  env:
282
    CC: 'clang'
283
    CXX: 'clang++'
284
  commands:
285
  - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
286
EOF
287
fi
288

289
if [[ "${windows_projects}" != "" ]]; then
290
  cat <<EOF
291
- label: ':windows: Windows x64'
292
  artifact_paths:
293
  - 'artifacts/**/*'
294
  - '*_result.json'
295
  - 'build/test-results.xml'
296
  agents: ${WINDOWS_AGENTS}
297
  retry:
298
    automatic:
299
      - exit_status: -1  # Agent was lost
300
        limit: 2
301
      - exit_status: 255 # Forced agent shutdown
302
        limit: 2
303
  timeout_in_minutes: 150
304
  env:
305
    CC: 'cl'
306
    CXX: 'cl'
307
    LD: 'link'
308
  commands:
309
  - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
310
  - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
311
EOF
312
fi
313