chrome/installer/linux/sysroot_scripts/sysroot-creator.sh

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 #
   5 # This script should not be run directly but sourced by the other
   6 # scripts (e.g. sysroot-creator-trusty.sh).  Its up to the parent scripts
   7 # to define certain environment variables: e.g.
   8 #  DIST=trusty
   9 #  APT_REPO=http://archive.ubuntu.com/ubuntu
  10 #  KEYRING_FILE=/usr/share/keyrings/ubuntu-archive-keyring.gpg
  11 #  DEBIAN_PACKAGES="gcc libz libssl"
  12
  13 #@ This script builds a Debian sysroot images for building Google Chrome.
  14 #@
  15 #@  Generally this script is invoked as:
  16 #@  sysroot-creator-<flavour>.sh <mode> <args>*
  17 #@  Available modes are shown below.
  18 #@
  19 #@ List of modes:
  20
  21 ######################################################################
  22 # Config
  23 ######################################################################
  24
  25 set -o nounset
  26 set -o errexit
  27
  28 SCRIPT_DIR=$(cd $(dirname $0) && pwd)
  29
  30 if [ -z "${DIST:-}" ]; then
  31   echo "error: DIST not defined"
  32   exit 1
  33 fi
  34
  35 if [ -z "${APT_REPO:-}" ]; then
  36   echo "error: APT_REPO not defined"
  37   exit 1
  38 fi
  39
  40 if [ -z "${KEYRING_FILE:-}" ]; then
  41   echo "error: KEYRING_FILE not defined"
  42   exit 1
  43 fi
  44
  45 if [ -z "${DEBIAN_PACKAGES:-}" ]; then
  46   echo "error: DEBIAN_PACKAGES not defined"
  47   exit 1
  48 fi
  49
  50 readonly REPO_BASEDIR="${APT_REPO}/dists/${DIST}"
  51
  52 # This is where the staging sysroot is.
  53 readonly INSTALL_ROOT_AMD64=$(pwd)/${DIST}_amd64_staging
  54 readonly INSTALL_ROOT_I386=$(pwd)/${DIST}_i386_staging
  55 readonly INSTALL_ROOT_ARM=$(pwd)/${DIST}_arm_staging
  56
  57 readonly REQUIRED_TOOLS="wget"
  58
  59 ######################################################################
  60 # Package Config
  61 ######################################################################
  62
  63 readonly RELEASE_FILE="Release"
  64 readonly RELEASE_FILE_GPG="Release.gpg"
  65 readonly RELEASE_LIST="${REPO_BASEDIR}/${RELEASE_FILE}"
  66 readonly RELEASE_LIST_GPG="${REPO_BASEDIR}/${RELEASE_FILE_GPG}"
  67 readonly PACKAGE_FILE_AMD64="main/binary-amd64/Packages.bz2"
  68 readonly PACKAGE_FILE_I386="main/binary-i386/Packages.bz2"
  69 readonly PACKAGE_FILE_ARM="main/binary-armhf/Packages.bz2"
  70 readonly PACKAGE_LIST_AMD64="${REPO_BASEDIR}/${PACKAGE_FILE_AMD64}"
  71 readonly PACKAGE_LIST_I386="${REPO_BASEDIR}/${PACKAGE_FILE_I386}"
  72 readonly PACKAGE_LIST_ARM="${REPO_BASEDIR}/${PACKAGE_FILE_ARM}"
  73
  74 readonly DEBIAN_DEP_LIST_AMD64="packagelist.${DIST}.amd64"
  75 readonly DEBIAN_DEP_LIST_I386="packagelist.${DIST}.i386"
  76 readonly DEBIAN_DEP_LIST_ARM="packagelist.${DIST}.arm"
  77
  78 ######################################################################
  79 # Helper
  80 ######################################################################
  81
  82 Banner() {
  83   echo "######################################################################"
  84   echo $*
  85   echo "######################################################################"
  86 }
  87
  88
  89 SubBanner() {
  90   echo "----------------------------------------------------------------------"
  91   echo $*
  92   echo "----------------------------------------------------------------------"
  93 }
  94
  95
  96 Usage() {
  97   egrep "^#@" "${BASH_SOURCE[0]}" | cut --bytes=3-
  98 }
  99
 100
 101 DownloadOrCopy() {
 102   if [ -f "$2" ] ; then
 103     echo "$2 already in place"
 104     return
 105   fi
 106
 107   HTTP=0
 108   echo "$1" | grep -qs ^http:// && HTTP=1
 109   if [ "$HTTP" = "1" ]; then
 110     SubBanner "downloading from $1 -> $2"
 111     wget "$1" -O "${2}.partial"
 112     mv "${2}.partial" $2
 113   else
 114     SubBanner "copying from $1"
 115     cp "$1" "$2"
 116   fi
 117 }
 118
 119
 120 SetEnvironmentVariables() {
 121   ARCH=""
 122   echo $1 | grep -qs Amd64$ && ARCH=AMD64
 123   if [ -z "$ARCH" ]; then
 124     echo $1 | grep -qs I386$ && ARCH=I386
 125   fi
 126   if [ -z "$ARCH" ]; then
 127     echo $1 | grep -qs ARM$ && ARCH=ARM
 128   fi
 129   case "$ARCH" in
 130     ARM)
 131       INSTALL_ROOT="$INSTALL_ROOT_ARM";
 132       ;;
 133     AMD64)
 134       INSTALL_ROOT="$INSTALL_ROOT_AMD64";
 135       ;;
 136     I386)
 137       INSTALL_ROOT="$INSTALL_ROOT_I386";
 138       ;;
 139     *)
 140       echo "ERROR: Unexpected bad architecture."
 141       exit 1
 142       ;;
 143   esac
 144 }
 145
 146
 147 # some sanity checks to make sure this script is run from the right place
 148 # with the right tools
 149 SanityCheck() {
 150   Banner "Sanity Checks"
 151
 152   if ! mkdir -p "${INSTALL_ROOT}" ; then
 153      echo "ERROR: ${INSTALL_ROOT} can't be created."
 154     exit 1
 155   fi
 156
 157   CHROME_DIR=$(cd "${SCRIPT_DIR}/../../.." && pwd)
 158   BUILD_DIR=${CHROME_DIR}/out/sysroot-build/${DIST}
 159   mkdir -p ${BUILD_DIR}
 160   echo "Using build directory: ${BUILD_DIR}"
 161
 162   for tool in ${REQUIRED_TOOLS} ; do
 163     if ! which ${tool} > /dev/null ; then
 164       echo "Required binary $tool not found."
 165       echo "Exiting."
 166       exit 1
 167     fi
 168   done
 169 }
 170
 171
 172 ChangeDirectory() {
 173   # Change directory to where this script is.
 174   cd ${SCRIPT_DIR}
 175 }
 176
 177
 178 ClearInstallDir() {
 179   Banner "Clearing dirs in ${INSTALL_ROOT}"
 180   rm -rf ${INSTALL_ROOT}/*
 181 }
 182
 183
 184 CreateTarBall() {
 185   local tarball=$1
 186   Banner "Creating tar ball ${tarball}"
 187   tar zcf ${tarball} -C ${INSTALL_ROOT} .
 188 }
 189
 190 CheckBuildSysrootArgs() {
 191   if [ "$#" -ne "1" ]; then
 192     echo "ERROR: BuildSysroot commands only take 1 argument"
 193     exit 1
 194   fi
 195
 196   if [ -z "$1" ]; then
 197     echo "ERROR: tarball name required"
 198     exit 1
 199   fi
 200 }
 201
 202 ExtractPackageBz2() {
 203   bzcat "$1" | egrep '^(Package:|Filename:|SHA256:) ' > "$2"
 204 }
 205
 206 GeneratePackageListAmd64() {
 207   local output_file="$1"
 208   local package_list="${BUILD_DIR}/Packages.${DIST}_amd64.bz2"
 209   local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64"
 210   DownloadOrCopy "${PACKAGE_LIST_AMD64}" "${package_list}"
 211   VerifyPackageListing "${PACKAGE_FILE_AMD64}" "${package_list}"
 212   ExtractPackageBz2 "$package_list" "$tmp_package_list"
 213   GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}
 214     ${DEBIAN_PACKAGES_X86}"
 215 }
 216
 217 GeneratePackageListI386() {
 218   local output_file="$1"
 219   local package_list="${BUILD_DIR}/Packages.${DIST}_i386.bz2"
 220   local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64"
 221   DownloadOrCopy "${PACKAGE_LIST_I386}" "${package_list}"
 222   VerifyPackageListing "${PACKAGE_FILE_I386}" "${package_list}"
 223   ExtractPackageBz2 "$package_list" "$tmp_package_list"
 224   GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}
 225     ${DEBIAN_PACKAGES_X86}"
 226 }
 227
 228 GeneratePackageListARM() {
 229   local output_file="$1"
 230   local package_list="${BUILD_DIR}/Packages.${DIST}_arm.bz2"
 231   local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_arm"
 232   DownloadOrCopy "${PACKAGE_LIST_ARM}" "${package_list}"
 233   VerifyPackageListing "${PACKAGE_FILE_ARM}" "${package_list}"
 234   ExtractPackageBz2 "$package_list" "$tmp_package_list"
 235   GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}"
 236 }
 237
 238 StripChecksumsFromPackageList() {
 239   local package_file="$1"
 240   sed -i 's/ [a-f0-9]\{64\}$//' "$package_file"
 241 }
 242
 243 VerifyPackageFilesMatch() {
 244   local downloaded_package_file="$1"
 245   local stored_package_file="$2"
 246   diff -u "$downloaded_package_file" "$stored_package_file"
 247   if [ "$?" -ne "0" ]; then
 248     echo "ERROR: downloaded package files does not match $2."
 249     echo "You may need to run UpdatePackageLists."
 250     exit 1
 251   fi
 252 }
 253
 254 ######################################################################
 255 #
 256 ######################################################################
 257
 258 HacksAndPatchesAmd64() {
 259   Banner "Misc Hacks & Patches"
 260   # these are linker scripts with absolute pathnames in them
 261   # which we rewrite here
 262   lscripts="${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libpthread.so \
 263             ${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libc.so"
 264
 265   #SubBanner "Rewriting Linker Scripts"
 266   sed -i -e 's|/usr/lib/x86_64-linux-gnu/||g'  ${lscripts}
 267   sed -i -e 's|/lib/x86_64-linux-gnu/||g' ${lscripts}
 268
 269   # This is for chrome's ./build/linux/pkg-config-wrapper
 270   # which overwrites PKG_CONFIG_PATH internally
 271   SubBanner "Package Configs Symlink"
 272   mkdir -p ${INSTALL_ROOT}/usr/share
 273   ln -s ../lib/x86_64-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
 274
 275   SubBanner "Adding an additional ld.conf include"
 276   LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf"
 277   echo /usr/lib/gcc/x86_64-linux-gnu/4.6 > "$LD_SO_HACK_CONF"
 278   echo /usr/lib >> "$LD_SO_HACK_CONF"
 279 }
 280
 281
 282 HacksAndPatchesI386() {
 283   Banner "Misc Hacks & Patches"
 284   # these are linker scripts with absolute pathnames in them
 285   # which we rewrite here
 286   lscripts="${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libpthread.so \
 287             ${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libc.so"
 288
 289   #SubBanner "Rewriting Linker Scripts"
 290   sed -i -e 's|/usr/lib/i386-linux-gnu/||g'  ${lscripts}
 291   sed -i -e 's|/lib/i386-linux-gnu/||g' ${lscripts}
 292
 293   # This is for chrome's ./build/linux/pkg-config-wrapper
 294   # which overwrites PKG_CONFIG_PATH internally
 295   SubBanner "Package Configs Symlink"
 296   mkdir -p ${INSTALL_ROOT}/usr/share
 297   ln -s ../lib/i386-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
 298
 299   SubBanner "Adding an additional ld.conf include"
 300   LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf"
 301   echo /usr/lib/gcc/i486-linux-gnu/4.6 > "$LD_SO_HACK_CONF"
 302   echo /usr/lib >> "$LD_SO_HACK_CONF"
 303 }
 304
 305
 306 HacksAndPatchesARM() {
 307   Banner "Misc Hacks & Patches"
 308   # these are linker scripts with absolute pathnames in them
 309   # which we rewrite here
 310   lscripts="${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libpthread.so \
 311             ${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libc.so"
 312
 313   #SubBanner "Rewriting Linker Scripts"
 314   sed -i -e 's|/usr/lib/arm-linux-gnueabihf/||g' ${lscripts}
 315   sed -i -e 's|/lib/arm-linux-gnueabihf/||g' ${lscripts}
 316
 317   # This is for chrome's ./build/linux/pkg-config-wrapper
 318   # which overwrites PKG_CONFIG_PATH internally
 319   SubBanner "Package Configs Symlink"
 320   mkdir -p ${INSTALL_ROOT}/usr/share
 321   ln -s ../lib/arm-linux-gnueabihf/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
 322 }
 323
 324
 325 InstallIntoSysroot() {
 326   Banner "Install Libs And Headers Into Jail"
 327
 328   mkdir -p ${BUILD_DIR}/debian-packages
 329   mkdir -p ${INSTALL_ROOT}
 330   while (( "$#" )); do
 331     local file="$1"
 332     local package="${BUILD_DIR}/debian-packages/${file##*/}"
 333     shift
 334     local sha256sum="$1"
 335     shift
 336     if [ "${#sha256sum}" -ne "64" ]; then
 337       echo "Bad sha256sum from package list"
 338       exit 1
 339     fi
 340
 341     Banner "Installing ${file}"
 342     DownloadOrCopy ${APT_REPO}/pool/${file} ${package}
 343     if [ ! -s "${package}" ] ; then
 344       echo
 345       echo "ERROR: bad package ${package}"
 346       exit 1
 347     fi
 348     echo "${sha256sum}  ${package}" | sha256sum --quiet -c
 349
 350     SubBanner "Extracting to ${INSTALL_ROOT}"
 351     dpkg --fsys-tarfile ${package}\
 352       | tar -xf - --exclude=./usr/share -C ${INSTALL_ROOT}
 353   done
 354 }
 355
 356
 357 CleanupJailSymlinks() {
 358   Banner "Jail symlink cleanup"
 359
 360   SAVEDPWD=$(pwd)
 361   cd ${INSTALL_ROOT}
 362   find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do
 363     # skip links with non-absolute paths
 364     echo "${target}" | grep -qs ^/ || continue
 365     echo "${link}: ${target}"
 366     case "${link}" in
 367       usr/lib/gcc/x86_64-linux-gnu/4.*/* | usr/lib/gcc/i486-linux-gnu/4.*/* | \
 368       usr/lib/gcc/arm-linux-gnueabihf/4.*/*)
 369         # Relativize the symlink.
 370         ln -snfv "../../../../..${target}" "${link}"
 371         ;;
 372       usr/lib/x86_64-linux-gnu/* | usr/lib/i386-linux-gnu/* | \
 373       usr/lib/arm-linux-gnueabihf/*)
 374         # Relativize the symlink.
 375         ln -snfv "../../..${target}" "${link}"
 376         ;;
 377       usr/lib/*)
 378         # Relativize the symlink.
 379         ln -snfv "../..${target}" "${link}"
 380         ;;
 381       lib64/* | lib/*)
 382         # Relativize the symlink.
 383         ln -snfv "..${target}" "${link}"
 384         ;;
 385     esac
 386   done
 387
 388   find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do
 389     # Make sure we catch new bad links.
 390     if [ ! -r "${link}" ]; then
 391       echo "ERROR: FOUND BAD LINK ${link}"
 392       ls -l ${link}
 393       exit 1
 394     fi
 395   done
 396   cd "$SAVEDPWD"
 397 }
 398
 399 #@
 400 #@ BuildSysrootAmd64 <tarball-name>
 401 #@
 402 #@    Build everything and package it
 403 BuildSysrootAmd64() {
 404   CheckBuildSysrootArgs $@
 405   ClearInstallDir
 406   local package_file="$BUILD_DIR/package_with_sha256sum_amd64"
 407   GeneratePackageListAmd64 "$package_file"
 408   local files_and_sha256sums="$(cat ${package_file})"
 409   StripChecksumsFromPackageList "$package_file"
 410   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_AMD64"
 411   InstallIntoSysroot ${files_and_sha256sums}
 412   CleanupJailSymlinks
 413   HacksAndPatchesAmd64
 414   CreateTarBall "$1"
 415 }
 416
 417 #@
 418 #@ BuildSysrootI386 <tarball-name>
 419 #@
 420 #@    Build everything and package it
 421 BuildSysrootI386() {
 422   CheckBuildSysrootArgs $@
 423   ClearInstallDir
 424   local package_file="$BUILD_DIR/package_with_sha256sum_i386"
 425   GeneratePackageListI386 "$package_file"
 426   local files_and_sha256sums="$(cat ${package_file})"
 427   StripChecksumsFromPackageList "$package_file"
 428   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_I386"
 429   InstallIntoSysroot ${files_and_sha256sums}
 430   CleanupJailSymlinks
 431   HacksAndPatchesI386
 432   CreateTarBall "$1"
 433 }
 434
 435 #@
 436 #@ BuildSysrootARM <tarball-name>
 437 #@
 438 #@    Build everything and package it
 439 BuildSysrootARM() {
 440   CheckBuildSysrootArgs $@
 441   ClearInstallDir
 442   local package_file="$BUILD_DIR/package_with_sha256sum_arm"
 443   GeneratePackageListARM "$package_file"
 444   local files_and_sha256sums="$(cat ${package_file})"
 445   StripChecksumsFromPackageList "$package_file"
 446   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM"
 447   APT_REPO=${APR_REPO_ARM:=$APT_REPO}
 448   InstallIntoSysroot ${files_and_sha256sums}
 449   CleanupJailSymlinks
 450   HacksAndPatchesARM
 451   CreateTarBall "$1"
 452 }
 453
 454 #
 455 # CheckForDebianGPGKeyring
 456 #
 457 #     Make sure the Debian GPG keys exist. Otherwise print a helpful message.
 458 #
 459 CheckForDebianGPGKeyring() {
 460   if [ ! -e "$KEYRING_FILE" ]; then
 461     echo "Debian GPG keys missing. Install the debian-archive-keyring package."
 462     exit 1
 463   fi
 464 }
 465
 466 #
 467 # VerifyPackageListing
 468 #
 469 #     Verifies the downloaded Packages.bz2 file has the right checksums.
 470 #
 471 VerifyPackageListing() {
 472   local file_path=$1
 473   local output_file=$2
 474   local release_file="${BUILD_DIR}/${RELEASE_FILE}"
 475   local release_file_gpg="${BUILD_DIR}/${RELEASE_FILE_GPG}"
 476
 477   CheckForDebianGPGKeyring
 478
 479   DownloadOrCopy ${RELEASE_LIST} ${release_file}
 480   DownloadOrCopy ${RELEASE_LIST_GPG} ${release_file_gpg}
 481   echo "Verifying: ${release_file} with ${release_file_gpg}"
 482   gpgv --keyring $KEYRING_FILE ${release_file_gpg} ${release_file}
 483
 484   echo "Verifying: ${output_file}"
 485   local checksums=$(grep ${file_path} ${release_file} | cut -d " " -f 2)
 486   local sha256sum=$(echo ${checksums} | cut -d " " -f 3)
 487
 488   if [ "${#sha256sum}" -ne "64" ]; then
 489     echo "Bad sha256sum from ${RELEASE_LIST}"
 490     exit 1
 491   fi
 492
 493   echo "${sha256sum}  ${output_file}" | sha256sum --quiet -c
 494 }
 495
 496 #
 497 # GeneratePackageList
 498 #
 499 #     Looks up package names in ${BUILD_DIR}/Packages and write list of URLs
 500 #     to output file.
 501 #
 502 GeneratePackageList() {
 503   local input_file="$1"
 504   local output_file="$2"
 505   echo "Updating: ${output_file} from ${input_file}"
 506   /bin/rm -f "${output_file}"
 507   shift
 508   shift
 509   for pkg in $@ ; do
 510     local pkg_full=$(grep -A 1 " ${pkg}\$" "$input_file" | \
 511       egrep -o "pool/.*")
 512     if [ -z "${pkg_full}" ]; then
 513         echo "ERROR: missing package: $pkg"
 514         exit 1
 515     fi
 516     local pkg_nopool=$(echo "$pkg_full" | sed "s/^pool\///")
 517     local sha256sum=$(grep -A 4 " ${pkg}\$" "$input_file" | \
 518       grep ^SHA256: | sed 's/^SHA256: //')
 519     if [ "${#sha256sum}" -ne "64" ]; then
 520       echo "Bad sha256sum from Packages"
 521       exit 1
 522     fi
 523     echo $pkg_nopool $sha256sum >> "$output_file"
 524   done
 525   # sort -o does an in-place sort of this file
 526   sort "$output_file" -o "$output_file"
 527 }
 528
 529 #@
 530 #@ UpdatePackageListsAmd64
 531 #@
 532 #@     Regenerate the package lists such that they contain an up-to-date
 533 #@     list of URLs within the Debian archive. (For amd64)
 534 UpdatePackageListsAmd64() {
 535   GeneratePackageListAmd64 "$DEBIAN_DEP_LIST_AMD64"
 536   StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_AMD64"
 537 }
 538
 539 #@
 540 #@ UpdatePackageListsI386
 541 #@
 542 #@     Regenerate the package lists such that they contain an up-to-date
 543 #@     list of URLs within the Debian archive. (For i386)
 544 UpdatePackageListsI386() {
 545   GeneratePackageListI386 "$DEBIAN_DEP_LIST_I386"
 546   StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_I386"
 547 }
 548
 549 #@
 550 #@ UpdatePackageListsARM
 551 #@
 552 #@     Regenerate the package lists such that they contain an up-to-date
 553 #@     list of URLs within the Debian archive. (For arm)
 554 UpdatePackageListsARM() {
 555   GeneratePackageListARM "$DEBIAN_DEP_LIST_ARM"
 556   StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_ARM"
 557 }
 558
 559 if [ $# -eq 0 ] ; then
 560   echo "ERROR: you must specify a mode on the commandline"
 561   echo
 562   Usage
 563   exit 1
 564 elif [ "$(type -t $1)" != "function" ]; then
 565   echo "ERROR: unknown function '$1'." >&2
 566   echo "For help, try:"
 567   echo "    $0 help"
 568   exit 1
 569 else
 570   ChangeDirectory
 571   SetEnvironmentVariables "$1"
 572   SanityCheck
 573   "$@"
 574 fi