llvm-project
312 строк · 7.8 Кб
1#!/usr/bin/env bash
2#===----------------------------------------------------------------------===##
3#
4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5# See https://llvm.org/LICENSE.txt for license information.
6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7#
8#===----------------------------------------------------------------------===##
9
10#
11# This file generates a Buildkite pipeline that triggers the various CI jobs for
12# the LLVM project during pre-commit CI.
13#
14# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
15#
16# As this outputs a yaml file, it's possible to log messages to stderr or
17# prefix with "#".
18
19
20set -eu21set -o pipefail22
23# Environment variables script works with:
24
25# Set by buildkite
26: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}27: ${BUILDKITE_COMMIT:=}28: ${BUILDKITE_BRANCH:=}29# Fetch origin to have an up to date merge base for the diff.
30git fetch origin31# List of files affected by this commit
32: ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}33# Filter rules for generic windows tests
34: ${WINDOWS_AGENTS:='{"queue": "windows"}'}35# Filter rules for generic linux tests
36: ${LINUX_AGENTS:='{"queue": "linux"}'}37
38reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"39if [[ "${reviewID}" != "" ]]; then40buildMessage="https://llvm.org/${reviewID}"41else
42buildMessage="Push to branch ${BUILDKITE_BRANCH}"43fi
44
45cat <<EOF46steps:
47EOF
48
49echo "Files modified:" >&250echo "$MODIFIED_FILES" >&251modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)52echo "Directories modified:" >&253echo "$modified_dirs" >&254
55function compute-projects-to-test() {56isForWindows=$157shift58projects=${@}59for project in ${projects}; do60echo "${project}"61case ${project} in62lld)63for p in bolt cross-project-tests; do64echo $p65done66;;67llvm)68for p in bolt clang clang-tools-extra lld lldb mlir polly; do69echo $p70done71# Flang is not stable in Windows CI at the moment72if [[ $isForWindows == 0 ]]; then73echo flang74fi75;;76clang)77# lldb is temporarily removed to alleviate Linux pre-commit CI waiting times78for p in clang-tools-extra compiler-rt cross-project-tests; do79echo $p80done81;;82clang-tools-extra)83echo libc84;;85mlir)86# Flang is not stable in Windows CI at the moment87if [[ $isForWindows == 0 ]]; then88echo flang89fi90;;91*)92# Nothing to do93;;94esac95done96}
97
98function compute-runtimes-to-test() {99projects=${@}100for project in ${projects}; do101case ${project} in102clang)103for p in libcxx libcxxabi libunwind; do104echo $p105done106;;107*)108# Nothing to do109;;110esac111done112}
113
114function add-dependencies() {115projects=${@}116for project in ${projects}; do117echo "${project}"118case ${project} in119bolt)120for p in clang lld llvm; do121echo $p122done123;;124cross-project-tests)125for p in lld clang; do126echo $p127done128;;129clang-tools-extra)130for p in llvm clang; do131echo $p132done133;;134compiler-rt|libc|openmp)135echo clang lld136;;137flang|lldb|libclc)138for p in llvm clang; do139echo $p140done141;;142lld|mlir|polly)143echo llvm144;;145*)146# Nothing to do147;;148esac149done150}
151
152function exclude-linux() {153projects=${@}154for project in ${projects}; do155case ${project} in156cross-project-tests) ;; # tests failing157openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410158*)159echo "${project}"160;;161esac162done163}
164
165function exclude-windows() {166projects=${@}167for project in ${projects}; do168case ${project} in169cross-project-tests) ;; # tests failing170compiler-rt) ;; # tests taking too long171openmp) ;; # TODO: having trouble with the Perl installation172libc) ;; # no Windows support173lldb) ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)174bolt) ;; # tests are not supported yet175*)176echo "${project}"177;;178esac179done180}
181
182# Prints only projects that are both present in $modified_dirs and the passed
183# list.
184function keep-modified-projects() {185projects=${@}186for project in ${projects}; do187if echo "$modified_dirs" | grep -q -E "^${project}$"; then188echo "${project}"189fi190done191}
192
193function check-targets() {194projects=${@}195for project in ${projects}; do196case ${project} in197clang-tools-extra)198echo "check-clang-tools"199;;200compiler-rt)201echo "check-all"202;;203cross-project-tests)204echo "check-cross-project"205;;206libcxx)207echo "check-cxx"208;;209libcxxabi)210echo "check-cxxabi"211;;212libunwind)213echo "check-unwind"214;;215lldb)216echo "check-lldb"217;;218pstl)219echo "check-all"220;;221libclc)222echo "check-all"223;;224*)225echo "check-${project}"226;;227esac228done229}
230
231# Project specific pipelines.
232
233# If libc++ or one of the runtimes directories changed.
234if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then235cat <<EOF236- trigger: "libcxx-ci"
237build:
238message: "${buildMessage}"239commit: "${BUILDKITE_COMMIT}"240branch: "${BUILDKITE_BRANCH}"241EOF
242fi
243
244# Generic pipeline for projects that have not defined custom steps.
245#
246# Individual projects should instead define the pre-commit CI tests that suits their
247# needs while letting them run on the infrastructure provided by LLVM.
248
249# Figure out which projects need to be built on each platform
250all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"251modified_projects="$(keep-modified-projects ${all_projects})"252
253linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))254linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)255linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)256
257linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})258linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)259linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)260
261windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))262windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)263windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)264
265# Generate the appropriate pipeline
266if [[ "${linux_projects}" != "" ]]; then267cat <<EOF268- label: ':linux: Linux x64'
269artifact_paths:
270- 'artifacts/**/*'
271- '*_result.json'
272- 'build/test-results.xml'
273agents: ${LINUX_AGENTS}274retry:
275automatic:
276- exit_status: -1 # Agent was lost
277limit: 2
278- exit_status: 255 # Forced agent shutdown
279limit: 2
280timeout_in_minutes: 120
281env:
282CC: 'clang'
283CXX: 'clang++'
284commands:
285- './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'286EOF
287fi
288
289if [[ "${windows_projects}" != "" ]]; then290cat <<EOF291- label: ':windows: Windows x64'
292artifact_paths:
293- 'artifacts/**/*'
294- '*_result.json'
295- 'build/test-results.xml'
296agents: ${WINDOWS_AGENTS}297retry:
298automatic:
299- exit_status: -1 # Agent was lost
300limit: 2
301- exit_status: 255 # Forced agent shutdown
302limit: 2
303timeout_in_minutes: 150
304env:
305CC: 'cl'
306CXX: 'cl'
307LD: 'link'
308commands:
309- 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'310- 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'311EOF
312fi
313