.ci/generate-buildkite-pipeline-premerge

   1 #!/usr/bin/env bash
   2 #===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 #===----------------------------------------------------------------------===##
   9
  10 #
  11 # This file generates a Buildkite pipeline that triggers the various CI jobs for
  12 # the LLVM project during pre-commit CI.
  13 #
  14 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
  15 #
  16 # As this outputs a yaml file, it's possible to log messages to stderr or
  17 # prefix with "#".
  18
  19
  20 set -eu
  21 set -o pipefail
  22
  23 # Environment variables script works with:
  24
  25 # Set by buildkite
  26 : ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
  27 : ${BUILDKITE_COMMIT:=}
  28 : ${BUILDKITE_BRANCH:=}
  29 # Fetch origin to have an up to date merge base for the diff.
  30 git fetch origin
  31 # List of files affected by this commit
  32 : ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
  33 # Filter rules for generic windows tests
  34 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
  35 # Filter rules for generic linux tests
  36 : ${LINUX_AGENTS:='{"queue": "linux"}'}
  37
  38 reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
  39 if [[ "${reviewID}" != "" ]]; then
  40   buildMessage="https://llvm.org/${reviewID}"
  41 else
  42   buildMessage="Push to branch ${BUILDKITE_BRANCH}"
  43 fi
  44
  45 cat <<EOF
  46 steps:
  47 EOF
  48
  49 echo "Files modified:" >&2
  50 echo "$MODIFIED_FILES" >&2
  51 modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
  52 echo "Directories modified:" >&2
  53 echo "$modified_dirs" >&2
  54
  55 function compute-projects-to-test() {
  56   isForWindows=$1
  57   shift
  58   projects=${@}
  59   for project in ${projects}; do
  60     echo "${project}"
  61     case ${project} in
  62     lld)
  63       for p in bolt cross-project-tests; do
  64         echo $p
  65       done
  66     ;;
  67     llvm)
  68       for p in bolt clang clang-tools-extra lld lldb mlir polly; do
  69         echo $p
  70       done
  71       # Flang is not stable in Windows CI at the moment
  72       if [[ $isForWindows == 0 ]]; then
  73         echo flang
  74       fi
  75     ;;
  76     clang)
  77       # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
  78       for p in clang-tools-extra compiler-rt cross-project-tests; do
  79         echo $p
  80       done
  81     ;;
  82     clang-tools-extra)
  83       echo libc
  84     ;;
  85     mlir)
  86       # Flang is not stable in Windows CI at the moment
  87       if [[ $isForWindows == 0 ]]; then
  88         echo flang
  89       fi
  90     ;;
  91     *)
  92       # Nothing to do
  93     ;;
  94     esac
  95   done
  96 }
  97
  98 function compute-runtimes-to-test() {
  99   projects=${@}
 100   for project in ${projects}; do
 101     case ${project} in
 102     clang)
 103       for p in libcxx libcxxabi libunwind; do
 104         echo $p
 105       done
 106     ;;
 107     *)
 108       # Nothing to do
 109     ;;
 110     esac
 111   done
 112 }
 113
 114 function add-dependencies() {
 115   projects=${@}
 116   for project in ${projects}; do
 117     echo "${project}"
 118     case ${project} in
 119     bolt)
 120       for p in clang lld llvm; do
 121         echo $p
 122       done
 123     ;;
 124     cross-project-tests)
 125       for p in lld clang; do
 126         echo $p
 127       done
 128     ;;
 129     clang-tools-extra)
 130       for p in llvm clang; do
 131         echo $p
 132       done
 133     ;;
 134     compiler-rt|libc|openmp)
 135       echo clang lld
 136     ;;
 137     flang|lldb|libclc)
 138       for p in llvm clang; do
 139         echo $p
 140       done
 141     ;;
 142     lld|mlir|polly)
 143       echo llvm
 144     ;;
 145     *)
 146       # Nothing to do
 147     ;;
 148     esac
 149   done
 150 }
 151
 152 function exclude-linux() {
 153   projects=${@}
 154   for project in ${projects}; do
 155     case ${project} in
 156     cross-project-tests) ;; # tests failing
 157     openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
 158     *)
 159       echo "${project}"
 160     ;;
 161     esac
 162   done
 163 }
 164
 165 function exclude-windows() {
 166   projects=${@}
 167   for project in ${projects}; do
 168     case ${project} in
 169     cross-project-tests) ;; # tests failing
 170     compiler-rt)         ;; # tests taking too long
 171     openmp)              ;; # TODO: having trouble with the Perl installation
 172     libc)                ;; # no Windows support
 173     lldb)                ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
 174     bolt)                ;; # tests are not supported yet
 175     *)
 176       echo "${project}"
 177     ;;
 178     esac
 179   done
 180 }
 181
 182 # Prints only projects that are both present in $modified_dirs and the passed
 183 # list.
 184 function keep-modified-projects() {
 185   projects=${@}
 186   for project in ${projects}; do
 187     if echo "$modified_dirs" | grep -q -E "^${project}$"; then
 188       echo "${project}"
 189     fi
 190   done
 191 }
 192
 193 function check-targets() {
 194   # Do not use "check-all" here because if there is "check-all" plus a
 195   # project specific target like "check-clang", that project's tests
 196   # will be run twice.
 197   projects=${@}
 198   for project in ${projects}; do
 199     case ${project} in
 200     clang-tools-extra)
 201       echo "check-clang-tools"
 202     ;;
 203     compiler-rt)
 204       echo "check-compiler-rt"
 205     ;;
 206     cross-project-tests)
 207       echo "check-cross-project"
 208     ;;
 209     libcxx)
 210       echo "check-cxx"
 211     ;;
 212     libcxxabi)
 213       echo "check-cxxabi"
 214     ;;
 215     libunwind)
 216       echo "check-unwind"
 217     ;;
 218     lldb)
 219       echo "check-lldb"
 220     ;;
 221     pstl)
 222       # Currently we do not run pstl tests in CI.
 223     ;;
 224     libclc)
 225       # Currently there is no testing for libclc.
 226     ;;
 227     *)
 228       echo "check-${project}"
 229     ;;
 230     esac
 231   done
 232 }
 233
 234 # Project specific pipelines.
 235
 236 # If libc++ or one of the runtimes directories changed.
 237 if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
 238   cat <<EOF
 239 - trigger: "libcxx-ci"
 240   build:
 241     message: "${buildMessage}"
 242     commit: "${BUILDKITE_COMMIT}"
 243     branch: "${BUILDKITE_BRANCH}"
 244 EOF
 245 fi
 246
 247 # Generic pipeline for projects that have not defined custom steps.
 248 #
 249 # Individual projects should instead define the pre-commit CI tests that suits their
 250 # needs while letting them run on the infrastructure provided by LLVM.
 251
 252 # Figure out which projects need to be built on each platform
 253 all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
 254 modified_projects="$(keep-modified-projects ${all_projects})"
 255
 256 linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
 257 linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
 258 linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
 259
 260 linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
 261 linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
 262 linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
 263
 264 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
 265 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
 266 windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 267
 268 # Generate the appropriate pipeline
 269 if [[ "${linux_projects}" != "" ]]; then
 270   cat <<EOF
 271 - label: ':linux: Linux x64'
 272   artifact_paths:
 273   - 'artifacts/**/*'
 274   - '*_result.json'
 275   - 'build/test-results.*.xml'
 276   agents: ${LINUX_AGENTS}
 277   retry:
 278     automatic:
 279       - exit_status: -1  # Agent was lost
 280         limit: 2
 281       - exit_status: 255 # Forced agent shutdown
 282         limit: 2
 283   timeout_in_minutes: 120
 284   env:
 285     CC: 'clang'
 286     CXX: 'clang++'
 287   commands:
 288   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
 289 EOF
 290 fi
 291
 292 if [[ "${windows_projects}" != "" ]]; then
 293   cat <<EOF
 294 - label: ':windows: Windows x64'
 295   artifact_paths:
 296   - 'artifacts/**/*'
 297   - '*_result.json'
 298   - 'build/test-results.*.xml'
 299   agents: ${WINDOWS_AGENTS}
 300   retry:
 301     automatic:
 302       - exit_status: -1  # Agent was lost
 303         limit: 2
 304       - exit_status: 255 # Forced agent shutdown
 305         limit: 2
 306   timeout_in_minutes: 150
 307   env:
 308     CC: 'cl'
 309     CXX: 'cl'
 310     LD: 'link'
 311   commands:
 312   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
 313   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 314 EOF
 315 fi