.ci/generate-buildkite-pipeline-premerge

   1 #!/usr/bin/env bash
   2 #===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 #===----------------------------------------------------------------------===##
   9
  10 #
  11 # This file generates a Buildkite pipeline that triggers the various CI jobs for
  12 # the LLVM project during pre-commit CI.
  13 #
  14 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
  15 #
  16 # As this outputs a yaml file, it's possible to log messages to stderr or
  17 # prefix with "#".
  18
  19
  20 set -eu
  21 set -o pipefail
  22
  23 # Environment variables script works with:
  24
  25 # Set by buildkite
  26 : ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
  27 : ${BUILDKITE_COMMIT:=}
  28 : ${BUILDKITE_BRANCH:=}
  29 # Fetch origin to have an up to date merge base for the diff.
  30 git fetch origin
  31 # List of files affected by this commit
  32 : ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
  33 # Filter rules for generic windows tests
  34 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
  35 # Filter rules for generic linux tests
  36 : ${LINUX_AGENTS:='{"queue": "linux"}'}
  37
  38 reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
  39 if [[ "${reviewID}" != "" ]]; then
  40   buildMessage="https://llvm.org/${reviewID}"
  41 else
  42   buildMessage="Push to branch ${BUILDKITE_BRANCH}"
  43 fi
  44
  45 cat <<EOF
  46 steps:
  47 EOF
  48
  49 echo "Files modified:" >&2
  50 echo "$MODIFIED_FILES" >&2
  51 modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
  52 echo "Directories modified:" >&2
  53 echo "$modified_dirs" >&2
  54
  55 function compute-projects-to-test() {
  56   isForWindows=$1
  57   shift
  58   projects=${@}
  59   for project in ${projects}; do
  60     echo "${project}"
  61     case ${project} in
  62     lld)
  63       for p in bolt cross-project-tests; do
  64         echo $p
  65       done
  66     ;;
  67     llvm)
  68       for p in bolt clang clang-tools-extra lld lldb mlir polly; do
  69         echo $p
  70       done
  71       # Flang is not stable in Windows CI at the moment
  72       if [[ $isForWindows == 0 ]]; then
  73         echo flang
  74       fi
  75     ;;
  76     clang)
  77       # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
  78       for p in clang-tools-extra compiler-rt cross-project-tests; do
  79         echo $p
  80       done
  81     ;;
  82     clang-tools-extra)
  83       echo libc
  84     ;;
  85     mlir)
  86       # Flang is not stable in Windows CI at the moment
  87       if [[ $isForWindows == 0 ]]; then
  88         echo flang
  89       fi
  90     ;;
  91     *)
  92       # Nothing to do
  93     ;;
  94     esac
  95   done
  96 }
  97
  98 function compute-runtimes-to-test() {
  99   projects=${@}
 100   for project in ${projects}; do
 101     case ${project} in
 102     clang)
 103       for p in libcxx libcxxabi libunwind; do
 104         echo $p
 105       done
 106     ;;
 107     *)
 108       # Nothing to do
 109     ;;
 110     esac
 111   done
 112 }
 113
 114 function add-dependencies() {
 115   projects=${@}
 116   for project in ${projects}; do
 117     echo "${project}"
 118     case ${project} in
 119     bolt)
 120       for p in clang lld llvm; do
 121         echo $p
 122       done
 123     ;;
 124     cross-project-tests)
 125       for p in lld clang; do
 126         echo $p
 127       done
 128     ;;
 129     clang-tools-extra)
 130       for p in llvm clang; do
 131         echo $p
 132       done
 133     ;;
 134     compiler-rt|libc|openmp)
 135       echo clang lld
 136     ;;
 137     flang|lldb|libclc)
 138       for p in llvm clang; do
 139         echo $p
 140       done
 141     ;;
 142     lld|mlir|polly)
 143       echo llvm
 144     ;;
 145     *)
 146       # Nothing to do
 147     ;;
 148     esac
 149   done
 150 }
 151
 152 function exclude-linux() {
 153   projects=${@}
 154   for project in ${projects}; do
 155     case ${project} in
 156     cross-project-tests) ;; # tests failing
 157     openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
 158     *)
 159       echo "${project}"
 160     ;;
 161     esac
 162   done
 163 }
 164
 165 function exclude-windows() {
 166   projects=${@}
 167   for project in ${projects}; do
 168     case ${project} in
 169     cross-project-tests) ;; # tests failing
 170     compiler-rt)         ;; # tests taking too long
 171     openmp)              ;; # TODO: having trouble with the Perl installation
 172     libc)                ;; # no Windows support
 173     lldb)                ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
 174     bolt)                ;; # tests are not supported yet
 175     *)
 176       echo "${project}"
 177     ;;
 178     esac
 179   done
 180 }
 181
 182 # Prints only projects that are both present in $modified_dirs and the passed
 183 # list.
 184 function keep-modified-projects() {
 185   projects=${@}
 186   for project in ${projects}; do
 187     if echo "$modified_dirs" | grep -q -E "^${project}$"; then
 188       echo "${project}"
 189     fi
 190   done
 191 }
 192
 193 function check-targets() {
 194   projects=${@}
 195   for project in ${projects}; do
 196     case ${project} in
 197     clang-tools-extra)
 198       echo "check-clang-tools"
 199     ;;
 200     compiler-rt)
 201       echo "check-all"
 202     ;;
 203     cross-project-tests)
 204       echo "check-cross-project"
 205     ;;
 206     libcxx)
 207       echo "check-cxx"
 208     ;;
 209     libcxxabi)
 210       echo "check-cxxabi"
 211     ;;
 212     libunwind)
 213       echo "check-unwind"
 214     ;;
 215     lldb)
 216       echo "check-lldb"
 217     ;;
 218     pstl)
 219       echo "check-all"
 220     ;;
 221     libclc)
 222       echo "check-all"
 223     ;;
 224     *)
 225       echo "check-${project}"
 226     ;;
 227     esac
 228   done
 229 }
 230
 231 # Project specific pipelines.
 232
 233 # If libc++ or one of the runtimes directories changed.
 234 if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
 235   cat <<EOF
 236 - trigger: "libcxx-ci"
 237   build:
 238     message: "${buildMessage}"
 239     commit: "${BUILDKITE_COMMIT}"
 240     branch: "${BUILDKITE_BRANCH}"
 241 EOF
 242 fi
 243
 244 # Generic pipeline for projects that have not defined custom steps.
 245 #
 246 # Individual projects should instead define the pre-commit CI tests that suits their
 247 # needs while letting them run on the infrastructure provided by LLVM.
 248
 249 # Figure out which projects need to be built on each platform
 250 all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
 251 modified_projects="$(keep-modified-projects ${all_projects})"
 252
 253 linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
 254 linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
 255 linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
 256
 257 linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
 258 linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
 259 linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
 260
 261 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
 262 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
 263 windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 264
 265 # Generate the appropriate pipeline
 266 if [[ "${linux_projects}" != "" ]]; then
 267   cat <<EOF
 268 - label: ':linux: Linux x64'
 269   artifact_paths:
 270   - 'artifacts/**/*'
 271   - '*_result.json'
 272   - 'build/test-results.xml'
 273   agents: ${LINUX_AGENTS}
 274   retry:
 275     automatic:
 276       - exit_status: -1  # Agent was lost
 277         limit: 2
 278       - exit_status: 255 # Forced agent shutdown
 279         limit: 2
 280   timeout_in_minutes: 120
 281   env:
 282     CC: 'clang'
 283     CXX: 'clang++'
 284   commands:
 285   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
 286 EOF
 287 fi
 288
 289 if [[ "${windows_projects}" != "" ]]; then
 290   cat <<EOF
 291 - label: ':windows: Windows x64'
 292   artifact_paths:
 293   - 'artifacts/**/*'
 294   - '*_result.json'
 295   - 'build/test-results.xml'
 296   agents: ${WINDOWS_AGENTS}
 297   retry:
 298     automatic:
 299       - exit_status: -1  # Agent was lost
 300         limit: 2
 301       - exit_status: 255 # Forced agent shutdown
 302         limit: 2
 303   timeout_in_minutes: 150
 304   env:
 305     CC: 'cl'
 306     CXX: 'cl'
 307     LD: 'link'
 308   commands:
 309   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
 310   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 311 EOF
 312 fi