.ci/generate-buildkite-pipeline-premerge

   1 #!/usr/bin/env bash
   2 #===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 #===----------------------------------------------------------------------===##
   9
  10 #
  11 # This file generates a Buildkite pipeline that triggers the various CI jobs for
  12 # the LLVM project during pre-commit CI.
  13 #
  14 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
  15 #
  16 # As this outputs a yaml file, it's possible to log messages to stderr or
  17 # prefix with "#".
  18
  19
  20 set -eu
  21 set -o pipefail
  22
  23 # Environment variables script works with:
  24
  25 # Set by buildkite
  26 : ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
  27 : ${BUILDKITE_COMMIT:=}
  28 : ${BUILDKITE_BRANCH:=}
  29 # Fetch origin to have an up to date merge base for the diff.
  30 git fetch origin
  31 # List of files affected by this commit
  32 : ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
  33 # Filter rules for generic windows tests
  34 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
  35 # Filter rules for generic linux tests
  36 : ${LINUX_AGENTS:='{"queue": "linux"}'}
  37
  38 reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
  39 if [[ "${reviewID}" != "" ]]; then
  40   buildMessage="https://llvm.org/${reviewID}"
  41 else
  42   buildMessage="Push to branch ${BUILDKITE_BRANCH}"
  43 fi
  44
  45 cat <<EOF
  46 steps:
  47 EOF
  48
  49 echo "Files modified:" >&2
  50 echo "$MODIFIED_FILES" >&2
  51 modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
  52 echo "Directories modified:" >&2
  53 echo "$modified_dirs" >&2
  54
  55 . ./.ci/compute-projects.sh
  56
  57 # Project specific pipelines.
  58
  59 # If libc++ or one of the runtimes directories changed.
  60 if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
  61   cat <<EOF
  62 - trigger: "libcxx-ci"
  63   build:
  64     message: "${buildMessage}"
  65     commit: "${BUILDKITE_COMMIT}"
  66     branch: "${BUILDKITE_BRANCH}"
  67 EOF
  68 fi
  69
  70 # Generic pipeline for projects that have not defined custom steps.
  71 #
  72 # Individual projects should instead define the pre-commit CI tests that suits their
  73 # needs while letting them run on the infrastructure provided by LLVM.
  74
  75 # Figure out which projects need to be built on each platform
  76 all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
  77 modified_projects="$(keep-modified-projects ${all_projects})"
  78
  79 linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
  80 linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
  81 linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
  82
  83 linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
  84 linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
  85 linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
  86
  87 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
  88 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
  89 windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
  90
  91 # Generate the appropriate pipeline
  92 if [[ "${linux_projects}" != "" ]]; then
  93   cat <<EOF
  94 - label: ':linux: Linux x64'
  95   artifact_paths:
  96   - 'artifacts/**/*'
  97   - '*_result.json'
  98   - 'build/test-results.*.xml'
  99   agents: ${LINUX_AGENTS}
 100   retry:
 101     automatic:
 102       - exit_status: -1  # Agent was lost
 103         limit: 2
 104       - exit_status: 255 # Forced agent shutdown
 105         limit: 2
 106   timeout_in_minutes: 120
 107   env:
 108     CC: 'clang'
 109     CXX: 'clang++'
 110   commands:
 111   - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"'
 112 EOF
 113 fi
 114
 115 if [[ "${windows_projects}" != "" ]]; then
 116   cat <<EOF
 117 - label: ':windows: Windows x64'
 118   artifact_paths:
 119   - 'artifacts/**/*'
 120   - '*_result.json'
 121   - 'build/test-results.*.xml'
 122   agents: ${WINDOWS_AGENTS}
 123   retry:
 124     automatic:
 125       - exit_status: -1  # Agent was lost
 126         limit: 2
 127       - exit_status: 255 # Forced agent shutdown
 128         limit: 2
 129   timeout_in_minutes: 150
 130   env:
 131     CC: 'cl'
 132     CXX: 'cl'
 133     LD: 'link'
 134   commands:
 135   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
 136   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 137 EOF
 138 fi