auto/build

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -x
   6 set -o pipefail
   7
   8 . "$(dirname "$0")/scripts/utils.sh"
   9
  10 # get $BUILD_BASENAME
  11 . tmp/build_environment
  12
  13 umask 022
  14
  15 ### Clone all output, from this point on, to the log file
  16
  17 BUILD_LOG="${BUILD_BASENAME}.buildlog"
  18 exec >  >(tee -a "$BUILD_LOG")
  19 # shellcheck disable=SC2064
  20 trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
  21 exec 2> >(tee -a "$BUILD_LOG" >&2)
  22 # shellcheck disable=SC2064
  23 trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
  24
  25 ### functions
  26
  27 print_iso_size () {
  28    local isofile="$1"
  29    [ -f "$isofile" ] || return 23
  30    size=$(stat --printf='%s' "$isofile")
  31    echo "I: The ISO is ${size} bytes large."
  32 }
  33
  34 ### Main
  35
  36 # we require building from git
  37 git rev-parse --is-inside-work-tree &> /dev/null \
  38    || fatal "${PWD} is not a Git tree."
  39
  40 . config/variables
  41
  42 # a clean starting point
  43 rm -rf cache/stages_rootfs
  44
  45 # get LB_BINARY_IMAGES
  46 . config/binary
  47
  48 # get LB_ARCHITECTURE and LB_DISTRIBUTION
  49 . config/bootstrap
  50
  51 # save variables that are needed by chroot_local-hooks
  52 {
  53    echo "KERNEL_VERSION=${KERNEL_VERSION}"
  54    echo "LB_DISTRIBUTION=${LB_DISTRIBUTION}"
  55    echo "POTFILES_DOT_IN='$(
  56            /bin/grep -E --no-filename '[^ #]*\.in$' po/POTFILES.in \
  57            | sed -e 's,^config/chroot_local-includes,,' | tr "\n" ' '
  58          )'"
  59 } >> config/chroot_local-includes/usr/share/tails/build/variables
  60
  61 # fix permissions on some source files that will be copied as is to the chroot.
  62 # they may be wrong, e.g. if the Git repository was cloned with a strict umask.
  63 chown    0:0   config/chroot_local-includes/etc/resolv.conf
  64 chmod -R go+rX config/binary_local-includes/
  65 chmod -R go+rX config/chroot_local-includes/etc
  66 chmod    0440  config/chroot_local-includes/etc/sudoers.d/*
  67 chmod    go+rX config/chroot_local-includes/lib
  68 chmod    go+rX config/chroot_local-includes/lib/live
  69 chmod -R go+rx config/chroot_local-includes/lib/live/config
  70 chmod    go+rX config/chroot_local-includes/lib/live/mount
  71 chmod -R go+rX config/chroot_local-includes/lib/systemd
  72 chmod    go+rX config/chroot_local-includes/live
  73 chmod -R go+rX config/chroot_local-includes/usr
  74 chmod -R go+rx config/chroot_local-includes/usr/local/bin
  75 chmod -R go+rx config/chroot_local-includes/usr/local/sbin
  76 chmod -R go+rX config/chroot_local-includes/usr/share/doc
  77 chmod -R go+rX config/chroot_local-includes/var
  78 chmod -R go+rX config/chroot_apt
  79 chmod -R go+rX config/chroot_sources
  80
  81 # normalize file timestamps
  82 find \
  83    config/binary_local-includes \
  84    config/chroot_local-includes \
  85    wiki/src \
  86    -exec touch --no-dereference --date="@$SOURCE_DATE_EPOCH" '{}' \;
  87
  88 # build the image
  89
  90 # we need /debootstrap/deburis to build a manifest of used packages:
  91 DEBOOTSTRAP_OPTIONS="${DEBOOTSTRAP_OPTIONS:-} --keep-debootstrap-dir"
  92
  93 # use our own APT repository's key:
  94 DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
  95 gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
  96     --no-tty \
  97     --import config/chroot_sources/tails.chroot.gpg
  98 DEBOOTSTRAP_GNUPG_KEYRING="$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.kbx"
  99 [ -e "$DEBOOTSTRAP_GNUPG_KEYRING" ] \
 100    || fatal "No debootstrap GnuPG keyring was created."
 101 DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_KEYRING"
 102
 103 export DEBOOTSTRAP_OPTIONS
 104
 105 # shellcheck disable=SC2223
 106 : ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K -no-exports'}
 107 MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -mem 512M -wildcards -ef chroot/usr/share/tails/build/mksquashfs-excludes"
 108 export MKSQUASHFS_OPTIONS
 109
 110 # build the doc wiki
 111 if [ "${TAILS_WEBSITE_CACHE:-0}" = 1 ]; then
 112    export WEBSITE_DEST_DIR=.
 113    export WEBSITE_CACHE_BASEDIR
 114
 115    echo "Website cache filesystem usage before garbage collection:"
 116    df "$WEBSITE_CACHE_BASEDIR"
 117    df --inodes "$WEBSITE_CACHE_BASEDIR"
 118
 119    website-cache gc
 120
 121    echo "Website cache filesystem usage after garbage collection:"
 122    df "$WEBSITE_CACHE_BASEDIR"
 123    df --inodes "$WEBSITE_CACHE_BASEDIR"
 124
 125    WEBSITE_CACHE_KEY=$(website-cache key)
 126    if ! website-cache get "$WEBSITE_CACHE_KEY"; then
 127       ./build-website
 128       WEBSITE_BUILD_DIR="config/chroot_local-includes/usr/share/doc/tails/website"
 129       echo "Website cache entry size: $(du --apparent-size --summarize "${WEBSITE_BUILD_DIR}" | cut -f1)"
 130       echo "Website cache entry inodes: $(du --inodes --summarize "${WEBSITE_BUILD_DIR}" | cut -f1)"
 131
 132       if ! website-cache put "$WEBSITE_CACHE_KEY"; then
 133          # Delete incomplete cache entry
 134          rm -rf "${WEBSITE_CACHE_BASEDIR:?}/${WEBSITE_CACHE_KEY:?}"
 135          cat <<EOF
 136 If this has started to fail with "No space left on device",
 137 check whether we're short on actual disk space or on inodes. And then:
 138
 139  - If we're short on actual disk space: have "website-cache gc" ensure
 140    that 10% disk space is available, by deleting as many cache entries
 141    as necessary, by decreasing age.
 142
 143  - Else, if we're short on inodes: either tune the website cache filesystem's
 144    inodes setup to match its actual (which is to store tons of small files),
 145    or migrate to storing cache entries as tarballs.
 146
 147 For more context, see #20150.
 148 EOF
 149          exit 1
 150       fi
 151       echo "Website cache filesystem usage after storing new cache entry:"
 152       df "$WEBSITE_CACHE_BASEDIR"
 153       df --inodes "$WEBSITE_CACHE_BASEDIR"
 154    fi
 155 else
 156    ./build-website
 157 fi
 158
 159 # refresh translations of our programs
 160 ./refresh-translations || fatal "refresh-translations failed ($?)."
 161
 162 # generate list of supported languages
 163 generate-languages-list || fatal "generate-languages-list failed ($?)."
 164
 165 BUILD_ISO_FILENAME="${BUILD_BASENAME}.iso"
 166 BUILD_MANIFEST="${BUILD_BASENAME}.build-manifest"
 167 BUILD_APT_SOURCES="${BUILD_BASENAME}.apt-sources"
 168 BUILD_PACKAGES="${BUILD_BASENAME}.packages"
 169 BUILD_USB_IMAGE_FILENAME="${BUILD_BASENAME}.img"
 170
 171 (
 172    echo "Mirrors:"
 173    apt-mirror debian
 174    apt-mirror debian-security
 175    apt-mirror torproject
 176    echo "Additional sources:"
 177    cat config/chroot_sources/*.chroot
 178 ) > "$BUILD_APT_SOURCES"
 179
 180 # make submodules available in the chroot:
 181 SUBMODULES_SRC="submodules/tails-workarounds"
 182 SUBMODULES_DST="config/chroot_local-includes/tmp/submodules"
 183 mkdir -p "$SUBMODULES_DST"
 184 # shellcheck disable=SC2086
 185 cp -a $SUBMODULES_SRC "$SUBMODULES_DST"/
 186
 187 # Inject the Tor Browser AppArmor policy into the chroot
 188 APPARMOR_D="config/chroot_local-includes/etc/apparmor.d"
 189 cp submodules/torbrowser-launcher/apparmor/torbrowser.Browser.firefox \
 190    "$APPARMOR_D"/
 191 cp submodules/torbrowser-launcher/apparmor/tunables/* \
 192    "$APPARMOR_D"/tunables/
 193
 194 echo "I: Building ISO image ${BUILD_ISO_FILENAME}..."
 195 time lb build noauto "${@}"
 196 BUILD_EXIT_CODE=$?
 197 [ -e binary.iso ] || fatal "lb build failed (${BUILD_EXIT_CODE})."
 198
 199 echo "I: ISO image was successfully created"
 200 print_iso_size binary.iso
 201
 202 echo "I: Hybriding it..."
 203 touch chroot/binary.iso
 204 mount --bind binary.iso chroot/binary.iso
 205 # shellcheck disable=SC2086
 206 chroot chroot isohybrid $ISOHYBRID_OPTS binary.iso || fatal "isohybrid failed"
 207 umount chroot/binary.iso
 208 print_iso_size binary.iso
 209 truncate -s %2048 binary.iso
 210 print_iso_size binary.iso
 211
 212 echo "I: Renaming generated files..."
 213 mv -i binary.iso "${BUILD_ISO_FILENAME}"
 214 mv -i binary.packages "${BUILD_PACKAGES}"
 215
 216 echo "I: Generating build manifest..."
 217 generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
 218
 219 echo "I: Creating USB image ${BUILD_USB_IMAGE_FILENAME}..."
 220 create-usb-image-from-iso "${BUILD_ISO_FILENAME}"