tools/clang/scripts/update.sh

   1 #!/usr/bin/env bash
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 # This script will check out llvm and clang into third_party/llvm and build it.
   7
   8 # Do NOT CHANGE this if you don't know what you're doing -- see
   9 # https://code.google.com/p/chromium/wiki/UpdatingClang
  10 # Reverting problematic clang rolls is safe, though.
  11 CLANG_REVISION=214024
  12
  13 THIS_DIR="$(dirname "${0}")"
  14 LLVM_DIR="${THIS_DIR}/../../../third_party/llvm"
  15 LLVM_BUILD_DIR="${LLVM_DIR}/../llvm-build"
  16 LLVM_BOOTSTRAP_DIR="${LLVM_DIR}/../llvm-bootstrap"
  17 LLVM_BOOTSTRAP_INSTALL_DIR="${LLVM_DIR}/../llvm-bootstrap-install"
  18 CLANG_DIR="${LLVM_DIR}/tools/clang"
  19 CLANG_TOOLS_EXTRA_DIR="${CLANG_DIR}/tools/extra"
  20 COMPILER_RT_DIR="${LLVM_DIR}/projects/compiler-rt"
  21 LIBCXX_DIR="${LLVM_DIR}/projects/libcxx"
  22 LIBCXXABI_DIR="${LLVM_DIR}/projects/libcxxabi"
  23 ANDROID_NDK_DIR="${LLVM_DIR}/../android_tools/ndk"
  24 STAMP_FILE="${LLVM_BUILD_DIR}/cr_build_revision"
  25
  26 ABS_LIBCXX_DIR="${PWD}/${LIBCXX_DIR}"
  27 ABS_LIBCXXABI_DIR="${PWD}/${LIBCXXABI_DIR}"
  28
  29
  30 # Use both the clang revision and the plugin revisions to test for updates.
  31 BLINKGCPLUGIN_REVISION=\
  32 $(grep LIBRARYNAME "$THIS_DIR"/../blink_gc_plugin/Makefile \
  33     | cut -d '_' -f 2)
  34 CLANG_AND_PLUGINS_REVISION="${CLANG_REVISION}-${BLINKGCPLUGIN_REVISION}"
  35
  36 # ${A:-a} returns $A if it's set, a else.
  37 LLVM_REPO_URL=${LLVM_URL:-https://llvm.org/svn/llvm-project}
  38
  39 if [[ -z "$GYP_DEFINES" ]]; then
  40   GYP_DEFINES=
  41 fi
  42 if [[ -z "$GYP_GENERATORS" ]]; then
  43   GYP_GENERATORS=
  44 fi
  45
  46
  47 # Die if any command dies, error on undefined variable expansions.
  48 set -eu
  49
  50 OS="$(uname -s)"
  51
  52 # Parse command line options.
  53 if_needed=
  54 force_local_build=
  55 run_tests=
  56 bootstrap=
  57 with_android=yes
  58 chrome_tools="plugins blink_gc_plugin"
  59 gcc_toolchain=
  60
  61 if [[ "${OS}" = "Darwin" ]]; then
  62   with_android=
  63 fi
  64 if [ "${OS}" = "FreeBSD" ]; then
  65   MAKE=gmake
  66 else
  67   MAKE=make
  68 fi
  69
  70 while [[ $# > 0 ]]; do
  71   case $1 in
  72     --bootstrap)
  73       bootstrap=yes
  74       ;;
  75     --if-needed)
  76       if_needed=yes
  77       ;;
  78     --force-local-build)
  79       force_local_build=yes
  80       ;;
  81     --print-revision)
  82       echo $CLANG_REVISION
  83       exit 0
  84       ;;
  85     --run-tests)
  86       run_tests=yes
  87       ;;
  88     --without-android)
  89       with_android=
  90       ;;
  91     --with-chrome-tools)
  92       shift
  93       if [[ $# == 0 ]]; then
  94         echo "--with-chrome-tools requires an argument."
  95         exit 1
  96       fi
  97       chrome_tools=$1
  98       ;;
  99     --gcc-toolchain)
 100       shift
 101       if [[ $# == 0 ]]; then
 102         echo "--gcc-toolchain requires an argument."
 103         exit 1
 104       fi
 105       if [[ -x "$1/bin/gcc" ]]; then
 106         gcc_toolchain=$1
 107       else
 108         echo "Invalid --gcc-toolchain: '$1'."
 109         echo "'$1/bin/gcc' does not appear to be valid."
 110         exit 1
 111       fi
 112       ;;
 113
 114     --help)
 115       echo "usage: $0 [--force-local-build] [--if-needed] [--run-tests] "
 116       echo "--bootstrap: First build clang with CC, then with itself."
 117       echo "--force-local-build: Don't try to download prebuilt binaries."
 118       echo "--if-needed: Download clang only if the script thinks it is needed."
 119       echo "--run-tests: Run tests after building. Only for local builds."
 120       echo "--print-revision: Print current clang revision and exit."
 121       echo "--without-android: Don't build ASan Android runtime library."
 122       echo "--with-chrome-tools: Select which chrome tools to build." \
 123            "Defaults to plugins."
 124       echo "    Example: --with-chrome-tools 'plugins empty-string'"
 125       echo "--gcc-toolchain: Set the prefix for which GCC version should"
 126       echo "    be used for building. For example, to use gcc in"
 127       echo "    /opt/foo/bin/gcc, use '--gcc-toolchain '/opt/foo"
 128       echo
 129       exit 1
 130       ;;
 131     *)
 132       echo "Unknown argument: '$1'."
 133       echo "Use --help for help."
 134       exit 1
 135       ;;
 136   esac
 137   shift
 138 done
 139
 140 if [[ -n "$if_needed" ]]; then
 141   if [[ "${OS}" == "Darwin" ]]; then
 142     # clang is used on Mac.
 143     true
 144   elif [[ "$GYP_DEFINES" =~ .*(clang|tsan|asan|lsan|msan)=1.* ]]; then
 145     # clang requested via $GYP_DEFINES.
 146     true
 147   elif [[ -d "${LLVM_BUILD_DIR}" ]]; then
 148     # clang previously downloaded, remove third_party/llvm-build to prevent
 149     # updating.
 150     true
 151   elif [[ "${OS}" == "Linux" ]]; then
 152     # Temporarily use clang on linux. Leave a stamp file behind, so that
 153     # this script can remove clang again on machines where it was autoinstalled.
 154     mkdir -p "${LLVM_BUILD_DIR}"
 155     touch "${LLVM_BUILD_DIR}/autoinstall_stamp"
 156     true
 157   else
 158     # clang wasn't needed, not doing anything.
 159     exit 0
 160   fi
 161 fi
 162
 163
 164 # Check if there's anything to be done, exit early if not.
 165 if [[ -f "${STAMP_FILE}" ]]; then
 166   PREVIOUSLY_BUILT_REVISON=$(cat "${STAMP_FILE}")
 167   if [[ -z "$force_local_build" ]] && \
 168        [[ "${PREVIOUSLY_BUILT_REVISON}" = \
 169           "${CLANG_AND_PLUGINS_REVISION}" ]]; then
 170     echo "Clang already at ${CLANG_AND_PLUGINS_REVISION}"
 171     exit 0
 172   fi
 173 fi
 174 # To always force a new build if someone interrupts their build half way.
 175 rm -f "${STAMP_FILE}"
 176
 177
 178 if [[ -z "$force_local_build" ]]; then
 179   # Check if there's a prebuilt binary and if so just fetch that. That's faster,
 180   # and goma relies on having matching binary hashes on client and server too.
 181   CDS_URL=https://commondatastorage.googleapis.com/chromium-browser-clang
 182   CDS_FILE="clang-${CLANG_REVISION}.tgz"
 183   CDS_OUT_DIR=$(mktemp -d -t clang_download.XXXXXX)
 184   CDS_OUTPUT="${CDS_OUT_DIR}/${CDS_FILE}"
 185   if [ "${OS}" = "Linux" ]; then
 186     CDS_FULL_URL="${CDS_URL}/Linux_x64/${CDS_FILE}"
 187   elif [ "${OS}" = "Darwin" ]; then
 188     CDS_FULL_URL="${CDS_URL}/Mac/${CDS_FILE}"
 189   fi
 190   echo Trying to download prebuilt clang
 191   if which curl > /dev/null; then
 192     curl -L --fail "${CDS_FULL_URL}" -o "${CDS_OUTPUT}" || \
 193         rm -rf "${CDS_OUT_DIR}"
 194   elif which wget > /dev/null; then
 195     wget "${CDS_FULL_URL}" -O "${CDS_OUTPUT}" || rm -rf "${CDS_OUT_DIR}"
 196   else
 197     echo "Neither curl nor wget found. Please install one of these."
 198     exit 1
 199   fi
 200   if [ -f "${CDS_OUTPUT}" ]; then
 201     rm -rf "${LLVM_BUILD_DIR}/Release+Asserts"
 202     mkdir -p "${LLVM_BUILD_DIR}/Release+Asserts"
 203     tar -xzf "${CDS_OUTPUT}" -C "${LLVM_BUILD_DIR}/Release+Asserts"
 204     echo clang "${CLANG_REVISION}" unpacked
 205     echo "${CLANG_AND_PLUGINS_REVISION}" > "${STAMP_FILE}"
 206     rm -rf "${CDS_OUT_DIR}"
 207     exit 0
 208   else
 209     echo Did not find prebuilt clang at r"${CLANG_REVISION}", building
 210   fi
 211 fi
 212
 213 if [[ -n "${with_android}" ]] && ! [[ -d "${ANDROID_NDK_DIR}" ]]; then
 214   echo "Android NDK not found at ${ANDROID_NDK_DIR}"
 215   echo "The Android NDK is needed to build a Clang whose -fsanitize=address"
 216   echo "works on Android. See "
 217   echo "http://code.google.com/p/chromium/wiki/AndroidBuildInstructions for how"
 218   echo "to install the NDK, or pass --without-android."
 219   exit 1
 220 fi
 221
 222 echo Getting LLVM r"${CLANG_REVISION}" in "${LLVM_DIR}"
 223 if ! svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" \
 224                     "${LLVM_DIR}"; then
 225   echo Checkout failed, retrying
 226   rm -rf "${LLVM_DIR}"
 227   svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" "${LLVM_DIR}"
 228 fi
 229
 230 echo Getting clang r"${CLANG_REVISION}" in "${CLANG_DIR}"
 231 svn co --force "${LLVM_REPO_URL}/cfe/trunk@${CLANG_REVISION}" "${CLANG_DIR}"
 232
 233 echo Getting compiler-rt r"${CLANG_REVISION}" in "${COMPILER_RT_DIR}"
 234 svn co --force "${LLVM_REPO_URL}/compiler-rt/trunk@${CLANG_REVISION}" \
 235                "${COMPILER_RT_DIR}"
 236
 237 # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
 238 # (i.e. this is needed for bootstrap builds).
 239 if [ "${OS}" = "Darwin" ]; then
 240   echo Getting libc++ r"${CLANG_REVISION}" in "${LIBCXX_DIR}"
 241   svn co --force "${LLVM_REPO_URL}/libcxx/trunk@${CLANG_REVISION}" \
 242                  "${LIBCXX_DIR}"
 243 fi
 244
 245 # While we're bundling our own libc++ on OS X, we need to compile libc++abi
 246 # into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
 247 if [ "${OS}" = "Darwin" ]; then
 248   echo Getting libc++abi r"${CLANG_REVISION}" in "${LIBCXXABI_DIR}"
 249   svn co --force "${LLVM_REPO_URL}/libcxxabi/trunk@${CLANG_REVISION}" \
 250                  "${LIBCXXABI_DIR}"
 251 fi
 252
 253 # Apply patch for test failing with --disable-pthreads (llvm.org/PR11974)
 254 pushd "${CLANG_DIR}"
 255 svn revert test/Index/crash-recovery-modules.m
 256 cat << 'EOF' |
 257 --- third_party/llvm/tools/clang/test/Index/crash-recovery-modules.m    (revision 202554)
 258 +++ third_party/llvm/tools/clang/test/Index/crash-recovery-modules.m    (working copy)
 259 @@ -12,6 +12,8 @@
 260
 261  // REQUIRES: crash-recovery
 262  // REQUIRES: shell
 263 +// XFAIL: *
 264 +//    (PR11974)
 265
 266  @import Crash;
 267 EOF
 268 patch -p4
 269 popd
 270
 271 # Echo all commands.
 272 set -x
 273
 274 NUM_JOBS=3
 275 if [[ "${OS}" = "Linux" ]]; then
 276   NUM_JOBS="$(grep -c "^processor" /proc/cpuinfo)"
 277 elif [ "${OS}" = "Darwin" -o "${OS}" = "FreeBSD" ]; then
 278   NUM_JOBS="$(sysctl -n hw.ncpu)"
 279 fi
 280
 281 if [[ -n "${gcc_toolchain}" ]]; then
 282   # Use the specified gcc installation for building.
 283   export CC="$gcc_toolchain/bin/gcc"
 284   export CXX="$gcc_toolchain/bin/g++"
 285   # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap compiler,
 286   # etc.) find the .so.
 287   export LD_LIBRARY_PATH="$(dirname $(${CXX} -print-file-name=libstdc++.so.6))"
 288 fi
 289
 290 export CFLAGS=""
 291 export CXXFLAGS=""
 292 # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
 293 # needed, on OS X it requires libc++. clang only automatically links to libc++
 294 # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run on
 295 # OS X versions as old as 10.7.
 296 # TODO(thakis): Some bots are still on 10.6, so for now bundle libc++.dylib.
 297 # Remove this once all bots are on 10.7+, then use --enable-libcpp=yes and
 298 # change all MACOSX_DEPLOYMENT_TARGET values to 10.7.
 299 if [ "${OS}" = "Darwin" ]; then
 300   # When building on 10.9, /usr/include usually doesn't exist, and while
 301   # Xcode's clang automatically sets a sysroot, self-built clangs don't.
 302   export CFLAGS="-isysroot $(xcrun --show-sdk-path)"
 303   export CPPFLAGS="${CFLAGS}"
 304   export CXXFLAGS="-stdlib=libc++ -nostdinc++ -I${ABS_LIBCXX_DIR}/include ${CFLAGS}"
 305 fi
 306
 307 # Build bootstrap clang if requested.
 308 if [[ -n "${bootstrap}" ]]; then
 309   ABS_INSTALL_DIR="${PWD}/${LLVM_BOOTSTRAP_INSTALL_DIR}"
 310   echo "Building bootstrap compiler"
 311   mkdir -p "${LLVM_BOOTSTRAP_DIR}"
 312   pushd "${LLVM_BOOTSTRAP_DIR}"
 313   if [[ ! -f ./config.status ]]; then
 314     # The bootstrap compiler only needs to be able to build the real compiler,
 315     # so it needs no cross-compiler output support. In general, the host
 316     # compiler should be as similar to the final compiler as possible, so do
 317     # keep --disable-threads & co.
 318     ../llvm/configure \
 319         --enable-optimized \
 320         --enable-targets=host-only \
 321         --enable-libedit=no \
 322         --disable-threads \
 323         --disable-pthreads \
 324         --without-llvmgcc \
 325         --without-llvmgxx \
 326         --prefix="${ABS_INSTALL_DIR}"
 327   fi
 328
 329   ${MAKE} -j"${NUM_JOBS}"
 330   if [[ -n "${run_tests}" ]]; then
 331     ${MAKE} check-all
 332   fi
 333
 334   ${MAKE} install
 335   if [[ -n "${gcc_toolchain}" ]]; then
 336     # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
 337     # compiler can start.
 338     cp -v "$(${CXX} -print-file-name=libstdc++.so.6)" \
 339       "${ABS_INSTALL_DIR}/lib/"
 340   fi
 341
 342   popd
 343   export CC="${ABS_INSTALL_DIR}/bin/clang"
 344   export CXX="${ABS_INSTALL_DIR}/bin/clang++"
 345
 346   if [[ -n "${gcc_toolchain}" ]]; then
 347     # Tell the bootstrap compiler to use a specific gcc prefix to search
 348     # for standard library headers and shared object file.
 349     export CFLAGS="--gcc-toolchain=${gcc_toolchain}"
 350     export CXXFLAGS="--gcc-toolchain=${gcc_toolchain}"
 351   fi
 352
 353   echo "Building final compiler"
 354 fi
 355
 356 # Build clang (in a separate directory).
 357 # The clang bots have this path hardcoded in built/scripts/slave/compile.py,
 358 # so if you change it you also need to change these links.
 359 mkdir -p "${LLVM_BUILD_DIR}"
 360 pushd "${LLVM_BUILD_DIR}"
 361
 362 # Build libc++.dylib while some bots are still on OS X 10.6.
 363 if [ "${OS}" = "Darwin" ]; then
 364   rm -rf libcxxbuild
 365   LIBCXXFLAGS="-O3 -std=c++11 -fstrict-aliasing"
 366
 367   # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
 368   # into different subdirectories.
 369   mkdir -p libcxxbuild/libcxx
 370   pushd libcxxbuild/libcxx
 371   ${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXX_DIR}"/src/*.cpp
 372   popd
 373
 374   mkdir -p libcxxbuild/libcxxabi
 375   pushd libcxxbuild/libcxxabi
 376   ${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXXABI_DIR}"/src/*.cpp -I"${ABS_LIBCXXABI_DIR}/include"
 377   popd
 378
 379   pushd libcxxbuild
 380   ${CC:-cc} libcxx/*.o libcxxabi/*.o -o libc++.1.dylib -dynamiclib \
 381     -nodefaultlibs -current_version 1 -compatibility_version 1 \
 382     -lSystem -install_name @executable_path/libc++.dylib \
 383     -Wl,-unexported_symbols_list,${ABS_LIBCXX_DIR}/lib/libc++unexp.exp \
 384     -Wl,-force_symbols_not_weak_list,${ABS_LIBCXX_DIR}/lib/notweak.exp \
 385     -Wl,-force_symbols_weak_list,${ABS_LIBCXX_DIR}/lib/weak.exp
 386   ln -sf libc++.1.dylib libc++.dylib
 387   popd
 388   export LDFLAGS+="-stdlib=libc++ -L${PWD}/libcxxbuild"
 389 fi
 390
 391 if [[ ! -f ./config.status ]]; then
 392   ../llvm/configure \
 393       --enable-optimized \
 394       --enable-libedit=no \
 395       --disable-threads \
 396       --disable-pthreads \
 397       --without-llvmgcc \
 398       --without-llvmgxx
 399 fi
 400
 401 if [[ -n "${gcc_toolchain}" ]]; then
 402   # Copy in the right stdlibc++.so.6 so clang can start.
 403   mkdir -p Release+Asserts/lib
 404   cp -v "$(${CXX} ${CXXFLAGS} -print-file-name=libstdc++.so.6)" \
 405     Release+Asserts/lib/
 406 fi
 407 MACOSX_DEPLOYMENT_TARGET=10.5 ${MAKE} -j"${NUM_JOBS}"
 408 STRIP_FLAGS=
 409 if [ "${OS}" = "Darwin" ]; then
 410   # See http://crbug.com/256342
 411   STRIP_FLAGS=-x
 412
 413   cp libcxxbuild/libc++.1.dylib Release+Asserts/bin
 414 fi
 415 strip ${STRIP_FLAGS} Release+Asserts/bin/clang
 416 popd
 417
 418 if [[ -n "${with_android}" ]]; then
 419   # Make a standalone Android toolchain.
 420   ${ANDROID_NDK_DIR}/build/tools/make-standalone-toolchain.sh \
 421       --platform=android-14 \
 422       --install-dir="${LLVM_BUILD_DIR}/android-toolchain" \
 423       --system=linux-x86_64 \
 424       --stl=stlport
 425
 426   # Android NDK r9d copies a broken unwind.h into the toolchain, see
 427   # http://crbug.com/357890
 428   rm -v "${LLVM_BUILD_DIR}"/android-toolchain/include/c++/*/unwind.h
 429
 430   # Build ASan runtime for Android.
 431   # Note: LLVM_ANDROID_TOOLCHAIN_DIR is not relative to PWD, but to where we
 432   # build the runtime, i.e. third_party/llvm/projects/compiler-rt.
 433   pushd "${LLVM_BUILD_DIR}"
 434   ${MAKE} -C tools/clang/runtime/ \
 435     LLVM_ANDROID_TOOLCHAIN_DIR="../../../llvm-build/android-toolchain"
 436   popd
 437 fi
 438
 439 # Build Chrome-specific clang tools. Paths in this list should be relative to
 440 # tools/clang.
 441 # For each tool directory, copy it into the clang tree and use clang's build
 442 # system to compile it.
 443 for CHROME_TOOL_DIR in ${chrome_tools}; do
 444   TOOL_SRC_DIR="${THIS_DIR}/../${CHROME_TOOL_DIR}"
 445   TOOL_DST_DIR="${LLVM_DIR}/tools/clang/tools/chrome-${CHROME_TOOL_DIR}"
 446   TOOL_BUILD_DIR="${LLVM_BUILD_DIR}/tools/clang/tools/chrome-${CHROME_TOOL_DIR}"
 447   rm -rf "${TOOL_DST_DIR}"
 448   cp -R "${TOOL_SRC_DIR}" "${TOOL_DST_DIR}"
 449   rm -rf "${TOOL_BUILD_DIR}"
 450   mkdir -p "${TOOL_BUILD_DIR}"
 451   cp "${TOOL_SRC_DIR}/Makefile" "${TOOL_BUILD_DIR}"
 452   MACOSX_DEPLOYMENT_TARGET=10.5 ${MAKE} -j"${NUM_JOBS}" -C "${TOOL_BUILD_DIR}"
 453 done
 454
 455 if [[ -n "$run_tests" ]]; then
 456   # Run a few tests.
 457   for CHROME_TOOL_DIR in ${chrome_tools}; do
 458     TOOL_SRC_DIR="${THIS_DIR}/../${CHROME_TOOL_DIR}"
 459     if [[ -f "${TOOL_SRC_DIR}/tests/test.sh" ]]; then
 460       "${TOOL_SRC_DIR}/tests/test.sh" "${LLVM_BUILD_DIR}/Release+Asserts"
 461     fi
 462   done
 463   pushd "${LLVM_BUILD_DIR}"
 464   ${MAKE} check-all
 465   popd
 466 fi
 467
 468 # After everything is done, log success for this revision.
 469 echo "${CLANG_AND_PLUGINS_REVISION}" > "${STAMP_FILE}"