.ci/generate-buildkite-pipeline-premerge

   1 #!/usr/bin/env bash
   2 #===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 #===----------------------------------------------------------------------===##
   9
  10 #
  11 # This file generates a Buildkite pipeline that triggers the various CI jobs for
  12 # the LLVM project during pre-commit CI (each time a Phabricator diff is uploaded).
  13 #
  14 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
  15 #
  16 # As this outputs a yaml file, it's possible to log messages to stderr or
  17 # prefix with "#".
  18
  19
  20 set -eu
  21 set -o pipefail
  22
  23 # Environment variables script works with:
  24
  25 # Set by buildkite
  26 : ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
  27 : ${BUILDKITE_COMMIT:=}
  28 : ${BUILDKITE_BRANCH:=}
  29 # Fetch origin to have an up to date merge base for the diff.
  30 git fetch origin
  31 # List of files affected by this commit
  32 : ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
  33 # Filter rules for generic windows tests
  34 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
  35 # Filter rules for generic linux tests
  36 : ${LINUX_AGENTS:='{"queue": "linux"}'}
  37 # Service agents, for interacting with Phabricator.
  38 : ${SERVICE_AGENTS:='{"queue": "service"}'}
  39
  40 reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
  41 if [[ "${reviewID}" != "" ]]; then
  42   buildMessage="https://llvm.org/${reviewID}"
  43 else
  44   buildMessage="Push to branch ${BUILDKITE_BRANCH}"
  45 fi
  46
  47 cat <<EOF
  48 steps:
  49 EOF
  50
  51 echo "Files modified:" >&2
  52 echo "$MODIFIED_FILES" >&2
  53 modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
  54 echo "Directories modified:" >&2
  55 echo "$modified_dirs" >&2
  56
  57 function compute-projects-to-test() {
  58   projects=${@}
  59   for project in ${projects}; do
  60     echo "${project}"
  61     case ${project} in
  62     lld)
  63       for p in bolt cross-project-tests; do
  64         echo $p
  65       done
  66     ;;
  67     llvm)
  68       for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
  69         echo $p
  70       done
  71     ;;
  72     clang)
  73       for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
  74         echo $p
  75       done
  76     ;;
  77     clang-tools-extra)
  78       echo libc
  79     ;;
  80     mlir)
  81       echo flang
  82     ;;
  83     *)
  84       # Nothing to do
  85     ;;
  86     esac
  87   done
  88 }
  89
  90 function add-dependencies() {
  91   projects=${@}
  92   for project in ${projects}; do
  93     echo "${project}"
  94     case ${project} in
  95     bolt)
  96       for p in lld llvm; do
  97         echo $p
  98       done
  99     ;;
 100     cross-project-tests)
 101       for p in lld clang; do
 102         echo $p
 103       done
 104     ;;
 105     clang-tools-extra)
 106       for p in llvm clang; do
 107         echo $p
 108       done
 109     ;;
 110     compiler-rt|libc|openmp)
 111       echo clang lld
 112     ;;
 113     flang|lldb)
 114       for p in llvm clang; do
 115         echo $p
 116       done
 117     ;;
 118     lld|mlir|polly)
 119       echo llvm
 120     ;;
 121     *)
 122       # Nothing to do
 123     ;;
 124     esac
 125   done
 126 }
 127
 128 function exclude-linux() {
 129   projects=${@}
 130   for project in ${projects}; do
 131     case ${project} in
 132     cross-project-tests) ;; # tests failing
 133     lldb)                ;; # tests failing
 134     openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
 135     *)
 136       echo "${project}"
 137     ;;
 138     esac
 139   done
 140 }
 141
 142 function exclude-windows() {
 143   projects=${@}
 144   for project in ${projects}; do
 145     case ${project} in
 146     cross-project-tests) ;; # tests failing
 147     compiler-rt)         ;; # tests taking too long
 148     openmp)              ;; # TODO: having trouble with the Perl installation
 149     libc)                ;; # no Windows support
 150     lldb)                ;; # tests failing
 151     bolt)                ;; # tests are not supported yet
 152     *)
 153       echo "${project}"
 154     ;;
 155     esac
 156   done
 157 }
 158
 159 # Prints only projects that are both present in $modified_dirs and the passed
 160 # list.
 161 function keep-modified-projects() {
 162   projects=${@}
 163   for project in ${projects}; do
 164     if echo "$modified_dirs" | grep -q -E "^${project}$"; then
 165       echo "${project}"
 166     fi
 167   done
 168 }
 169
 170 function check-targets() {
 171   projects=${@}
 172   for project in ${projects}; do
 173     case ${project} in
 174     clang-tools-extra)
 175       echo "check-clang-tools"
 176     ;;
 177     compiler-rt)
 178       echo "check-all"
 179     ;;
 180     cross-project-tests)
 181       echo "check-cross-project"
 182     ;;
 183     lldb)
 184       echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
 185     ;;
 186     pstl)
 187       echo "check-all"
 188     ;;
 189     libclc)
 190       echo "check-all"
 191     ;;
 192     *)
 193       echo "check-${project}"
 194     ;;
 195     esac
 196   done
 197 }
 198
 199 # Project specific pipelines.
 200
 201 # If libc++ or one of the runtimes directories changed.
 202 if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
 203   cat <<EOF
 204 - trigger: "libcxx-ci"
 205   build:
 206     message: "${buildMessage}"
 207     commit: "${BUILDKITE_COMMIT}"
 208     branch: "${BUILDKITE_BRANCH}"
 209 EOF
 210 fi
 211
 212 # If clang changed.
 213 if echo "$modified_dirs" | grep -q -E "^(clang)$"; then
 214   cat <<EOF
 215 - trigger: "clang-ci"
 216   build:
 217     message: "${buildMessage}"
 218     commit: "${BUILDKITE_COMMIT}"
 219     branch: "${BUILDKITE_BRANCH}"
 220 EOF
 221 fi
 222
 223 # Generic pipeline for projects that have not defined custom steps.
 224 #
 225 # Individual projects should instead define the pre-commit CI tests that suits their
 226 # needs while letting them run on the infrastructure provided by LLVM.
 227
 228 # Figure out which projects need to be built on each platform
 229 all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
 230 modified_projects="$(keep-modified-projects ${all_projects})"
 231
 232 linux_projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
 233 linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
 234 linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
 235
 236 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
 237 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
 238 windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 239
 240 # Generate the appropriate pipeline
 241 if [[ "${linux_projects}" != "" ]]; then
 242   cat <<EOF
 243 - label: ':linux: Linux x64'
 244   artifact_paths:
 245   - 'artifacts/**/*'
 246   - '*_result.json'
 247   - 'build/test-results.xml'
 248   agents: ${LINUX_AGENTS}
 249   retry:
 250     automatic:
 251       - exit_status: -1  # Agent was lost
 252         limit: 2
 253       - exit_status: 255 # Forced agent shutdown
 254         limit: 2
 255   timeout_in_minutes: 120
 256   env:
 257     CC: 'clang'
 258     CXX: 'clang++'
 259   commands:
 260   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"'
 261 EOF
 262 fi
 263
 264 if [[ "${windows_projects}" != "" ]]; then
 265   cat <<EOF
 266 - label: ':windows: Windows x64'
 267   artifact_paths:
 268   - 'artifacts/**/*'
 269   - '*_result.json'
 270   - 'build/test-results.xml'
 271   agents: ${WINDOWS_AGENTS}
 272   retry:
 273     automatic:
 274       - exit_status: -1  # Agent was lost
 275         limit: 2
 276       - exit_status: 255 # Forced agent shutdown
 277         limit: 2
 278   timeout_in_minutes: 150
 279   env:
 280     CC: 'cl'
 281     CXX: 'cl'
 282     LD: 'link'
 283   commands:
 284   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
 285   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 286 EOF
 287 fi
 288
 289 # If build was triggered from a Phabricator review - send an update back.
 290 if [[ -n "${ph_target_phid:-}" ]]; then
 291   cat << EOF
 292 - continue_on_failure: true
 293   wait: '~'
 294 - label: ':phabricator: update build status on Phabricator'
 295   agents: ${SERVICE_AGENTS}
 296   artifact_paths:
 297   - 'artifacts/**/*'
 298   commands:
 299   - export SRC=\$\${BUILDKITE_BUILD_PATH}/llvm-premerge-checks
 300   - rm -rf \$\${SRC}
 301   - git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "\$\${SRC}"
 302   - cd \$\${SRC}
 303   - git fetch origin "main":x
 304   - git checkout x
 305   - echo "llvm-premerge-checks commit"
 306   - git rev-parse HEAD
 307   - pip install -q -r \$\${SRC}/scripts/requirements.txt
 308   - cd "\$\$BUILDKITE_BUILD_CHECKOUT_PATH"
 309   - \$\${SRC}/scripts/summary.py
 310   timeout_in_minutes: 10
 311 EOF
 312 fi