.ci/generate-buildkite-pipeline-premerge

   1 #!/usr/bin/env bash
   2 #===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 #===----------------------------------------------------------------------===##
   9
  10 #
  11 # This file generates a Buildkite pipeline that triggers the various CI jobs for
  12 # the LLVM project during pre-commit CI (each time a Phabricator diff is uploaded).
  13 #
  14 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
  15 #
  16 # As this outputs a yaml file, it's possible to log messages to stderr or
  17 # prefix with "#".
  18
  19
  20 set -eu
  21 set -o pipefail
  22
  23 # Environment variables script works with:
  24
  25 # Fetch origin/main to have an up to date merge base for main...HEAD diff.
  26 git fetch origin main:main
  27 # List of files affected by this commit
  28 : ${MODIFIED_FILES:=$(git diff --name-only main...HEAD)}
  29 # Filter rules for generic windows tests
  30 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
  31 # Filter rules for generic linux tests
  32 : ${LINUX_AGENTS:='{"queue": "linux"}'}
  33 # Service agents, for interacting with Phabricator.
  34 : ${SERVICE_AGENTS:='{"queue": "service"}'}
  35 # Set by buildkite
  36 : ${BUILDKITE_COMMIT:=}
  37 : ${BUILDKITE_BRANCH:=}
  38
  39 reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
  40 if [[ "${reviewID}" != "" ]]; then
  41   buildMessage="https://llvm.org/${reviewID}"
  42 else
  43   buildMessage="Push to branch ${BUILDKITE_BRANCH}"
  44 fi
  45
  46 cat <<EOF
  47 steps:
  48 EOF
  49
  50 echo "Files modified:" >&2
  51 echo "$MODIFIED_FILES" >&2
  52 modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
  53 echo "Directories modified:" >&2
  54 echo "$modified_dirs" >&2
  55
  56 function compute-projects-to-test() {
  57   projects=${@}
  58   for project in ${projects}; do
  59     echo "${project}"
  60     case ${project} in
  61     lld)
  62       for p in bolt cross-project-tests; do
  63         echo $p
  64       done
  65     ;;
  66     llvm)
  67       for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
  68         echo $p
  69       done
  70     ;;
  71     clang)
  72       for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
  73         echo $p
  74       done
  75     ;;
  76     clang-tools-extra)
  77       echo libc
  78     ;;
  79     mlir)
  80       echo flang
  81     ;;
  82     *)
  83       # Nothing to do
  84     ;;
  85     esac
  86   done
  87 }
  88
  89 function add-dependencies() {
  90   projects=${@}
  91   for project in ${projects}; do
  92     echo "${project}"
  93     case ${project} in
  94     bolt)
  95       for p in lld llvm; do
  96         echo $p
  97       done
  98     ;;
  99     cross-project-tests)
 100       for p in lld clang; do
 101         echo $p
 102       done
 103     ;;
 104     clang-tools-extra)
 105       for p in llvm clang; do
 106         echo $p
 107       done
 108     ;;
 109     compiler-rt|libc|openmp)
 110       echo clang
 111     ;;
 112     flang|lldb)
 113       for p in llvm clang; do
 114         echo $p
 115       done
 116     ;;
 117     lld|mlir|polly)
 118       echo llvm
 119     ;;
 120     *)
 121       # Nothing to do
 122     ;;
 123     esac
 124   done
 125 }
 126
 127 function exclude-linux() {
 128   projects=${@}
 129   for project in ${projects}; do
 130     case ${project} in
 131     cross-project-tests) ;; # tests failing
 132     lldb)                ;; # tests failing
 133     openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
 134     *)
 135       echo "${project}"
 136     ;;
 137     esac
 138   done
 139 }
 140
 141 function exclude-windows() {
 142   projects=${@}
 143   for project in ${projects}; do
 144     case ${project} in
 145     cross-project-tests) ;; # tests failing
 146     compiler-rt)         ;; # tests taking too long
 147     openmp)              ;; # TODO: having trouble with the Perl installation
 148     libc)                ;; # no Windows support
 149     lldb)                ;; # tests failing
 150     bolt)                ;; # tests are not supported yet
 151     *)
 152       echo "${project}"
 153     ;;
 154     esac
 155   done
 156 }
 157
 158 # Prints only projects that are both present in $modified_dirs and the passed
 159 # list.
 160 function keep-modified-projects() {
 161   projects=${@}
 162   for project in ${projects}; do
 163     if echo "$modified_dirs" | grep -q -E "^${project}$"; then
 164       echo "${project}"
 165     fi
 166   done
 167 }
 168
 169 function check-targets() {
 170   projects=${@}
 171   for project in ${projects}; do
 172     case ${project} in
 173     clang-tools-extra)
 174       echo "check-clang-tools"
 175     ;;
 176     compiler-rt)
 177       echo "check-all"
 178     ;;
 179     cross-project-tests)
 180       echo "check-cross-project"
 181     ;;
 182     lldb)
 183       echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
 184     ;;
 185     pstl)
 186       echo "check-all"
 187     ;;
 188     libclc)
 189       echo "check-all"
 190     ;;
 191     *)
 192       echo "check-${project}"
 193     ;;
 194     esac
 195   done
 196 }
 197
 198 # Project specific pipelines.
 199
 200 # If libc++ or one of the runtimes directories changed.
 201 if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
 202   cat <<EOF
 203 - trigger: "libcxx-ci"
 204   build:
 205     message: "${buildMessage}"
 206     commit: "${BUILDKITE_COMMIT}"
 207     branch: "${BUILDKITE_BRANCH}"
 208 EOF
 209 fi
 210
 211 # If clang changed.
 212 if echo "$modified_dirs" | grep -q -E "^(clang)$"; then
 213   cat <<EOF
 214 - trigger: "clang-ci"
 215   build:
 216     message: "${buildMessage}"
 217     commit: "${BUILDKITE_COMMIT}"
 218     branch: "${BUILDKITE_BRANCH}"
 219 EOF
 220 fi
 221
 222 # Generic pipeline for projects that have not defined custom steps.
 223 #
 224 # Individual projects should instead define the pre-commit CI tests that suits their
 225 # needs while letting them run on the infrastructure provided by LLVM.
 226
 227 # Figure out which projects need to be built on each platform
 228 all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
 229 modified_projects="$(keep-modified-projects ${all_projects})"
 230
 231 linux_projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
 232 linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
 233 linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
 234
 235 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
 236 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
 237 windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 238
 239 # Generate the appropriate pipeline
 240 if [[ "${linux_projects}" != "" ]]; then
 241   cat <<EOF
 242 - label: ':linux: Linux x64'
 243   artifact_paths:
 244   - 'artifacts/**/*'
 245   - '*_result.json'
 246   - 'build/test-results.xml'
 247   agents: ${LINUX_AGENTS}
 248   retry:
 249     automatic:
 250       - exit_status: -1  # Agent was lost
 251         limit: 2
 252       - exit_status: 255 # Forced agent shutdown
 253         limit: 2
 254   timeout_in_minutes: 120
 255   env:
 256     CC: 'clang'
 257     CXX: 'clang++'
 258   commands:
 259   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"'
 260 EOF
 261 fi
 262
 263 if [[ "${windows_projects}" != "" ]]; then
 264   cat <<EOF
 265 - label: ':windows: Windows x64'
 266   artifact_paths:
 267   - 'artifacts/**/*'
 268   - '*_result.json'
 269   - 'build/test-results.xml'
 270   agents: ${WINDOWS_AGENTS}
 271   retry:
 272     automatic:
 273       - exit_status: -1  # Agent was lost
 274         limit: 2
 275       - exit_status: 255 # Forced agent shutdown
 276         limit: 2
 277   timeout_in_minutes: 150
 278   env:
 279     CC: 'cl'
 280     CXX: 'cl'
 281     LD: 'link'
 282   commands:
 283   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
 284   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 285 EOF
 286 fi
 287
 288 # If build was triggered from a Phabricator review - send an update back.
 289 if [[ -n "${ph_target_phid:-}" ]]; then
 290   cat << EOF
 291 - continue_on_failure: true
 292   wait: '~'
 293 - label: ':phabricator: update build status on Phabricator'
 294   agents: ${SERVICE_AGENTS}
 295   artifact_paths:
 296   - 'artifacts/**/*'
 297   commands:
 298   - export SRC=\$\${BUILDKITE_BUILD_PATH}/llvm-premerge-checks
 299   - rm -rf \$\${SRC}
 300   - git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "\$\${SRC}"
 301   - cd \$\${SRC}
 302   - git fetch origin "main":x
 303   - git checkout x
 304   - echo "llvm-premerge-checks commit"
 305   - git rev-parse HEAD
 306   - pip install -q -r \$\${SRC}/scripts/requirements.txt
 307   - cd "\$\$BUILDKITE_BUILD_CHECKOUT_PATH"
 308   - \$\${SRC}/scripts/summary.py
 309   timeout_in_minutes: 10
 310 EOF
 311 fi