auto/build

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -x
   6 set -o pipefail
   7
   8 . "$(dirname "$0")/scripts/utils.sh"
   9
  10 # get $BUILD_BASENAME
  11 . tmp/build_environment
  12
  13 umask 022
  14
  15 ### Clone all output, from this point on, to the log file
  16
  17 BUILD_LOG="${BUILD_BASENAME}.buildlog"
  18 exec > >(tee -a "$BUILD_LOG")
  19 # shellcheck disable=SC2064
  20 trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
  21 exec 2> >(tee -a "$BUILD_LOG" >&2)
  22 # shellcheck disable=SC2064
  23 trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
  24
  25 ### functions
  26
  27 print_iso_size() {
  28     local isofile="$1"
  29     [ -f "$isofile" ] || return 23
  30     size=$(stat --printf='%s' "$isofile")
  31     echo "I: The ISO is ${size} bytes large."
  32 }
  33
  34 ### Main
  35
  36 # we require building from git
  37 git rev-parse --is-inside-work-tree &>/dev/null ||
  38     fatal "${PWD} is not a Git tree."
  39
  40 . config/variables
  41
  42 # a clean starting point
  43 rm -rf cache/stages_rootfs
  44
  45 # get LB_BINARY_IMAGES
  46 . config/binary
  47
  48 # get LB_ARCHITECTURE and LB_DISTRIBUTION
  49 . config/bootstrap
  50
  51 # save variables that are needed by chroot_local-hooks
  52 {
  53     echo "KERNEL_VERSION=${KERNEL_VERSION}"
  54     echo "LB_DISTRIBUTION=${LB_DISTRIBUTION}"
  55     echo "POTFILES_DOT_IN='$(
  56         /bin/grep -E --no-filename '[^ #]*\.in$' po/POTFILES.in |
  57             sed -e 's,^config/chroot_local-includes,,' | tr "\n" ' '
  58     )'"
  59 } >>config/chroot_local-includes/usr/share/tails/build/variables
  60
  61 # fix permissions on some source files that will be copied as is to the chroot.
  62 # they may be wrong, e.g. if the Git repository was cloned with a strict umask.
  63 chown 0:0 config/chroot_local-includes/etc/resolv.conf
  64 chmod -R go+rX config/binary_local-includes/
  65 chmod -R go+rX config/chroot_local-includes/etc
  66 chmod 0440 config/chroot_local-includes/etc/sudoers.d/*
  67 chmod go+rX config/chroot_local-includes/lib
  68 chmod go+rX config/chroot_local-includes/lib/live
  69 chmod -R go+rx config/chroot_local-includes/lib/live/config
  70 chmod go+rX config/chroot_local-includes/lib/live/mount
  71 chmod -R go+rX config/chroot_local-includes/lib/systemd
  72 chmod go+rX config/chroot_local-includes/live
  73 chmod -R go+rX config/chroot_local-includes/usr
  74 chmod -R go+rx config/chroot_local-includes/usr/local/bin
  75 chmod -R go+rx config/chroot_local-includes/usr/local/sbin
  76 chmod -R go+rX config/chroot_local-includes/usr/share/doc
  77 chmod -R go+rX config/chroot_local-includes/var
  78 chmod -R go+rX config/chroot_apt
  79 chmod -R go+rX config/chroot_sources
  80
  81 # normalize file timestamps
  82 find \
  83     config/binary_local-includes \
  84     config/chroot_local-includes \
  85     wiki/src \
  86     -exec touch --no-dereference --date="@$SOURCE_DATE_EPOCH" '{}' \;
  87
  88 # build the image
  89
  90 # we need /debootstrap/deburis to build a manifest of used packages:
  91 DEBOOTSTRAP_OPTIONS="${DEBOOTSTRAP_OPTIONS:-} --keep-debootstrap-dir"
  92
  93 # use our own APT repository's key:
  94 DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
  95 gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
  96     --no-tty \
  97     --import config/chroot_sources/tails.chroot.gpg
  98 DEBOOTSTRAP_GNUPG_KEYRING="$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.kbx"
  99 [ -e "$DEBOOTSTRAP_GNUPG_KEYRING" ] ||
 100     fatal "No debootstrap GnuPG keyring was created."
 101 DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_KEYRING"
 102
 103 export DEBOOTSTRAP_OPTIONS
 104
 105 # shellcheck disable=SC2223
 106 : ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K -no-exports'}
 107 MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -mem 512M -wildcards -ef chroot/usr/share/tails/build/mksquashfs-excludes"
 108 export MKSQUASHFS_OPTIONS
 109
 110 # build the doc wiki
 111 if [ "${TAILS_WEBSITE_CACHE:-0}" = 1 ]; then
 112     export WEBSITE_DEST_DIR=.
 113     export WEBSITE_CACHE_BASEDIR
 114
 115     echo "Website cache filesystem usage before migration:"
 116     df "$WEBSITE_CACHE_BASEDIR"
 117     df --inodes "$WEBSITE_CACHE_BASEDIR"
 118
 119     website-cache migrate
 120
 121     echo "Website cache filesystem usage before garbage collection:"
 122     df "$WEBSITE_CACHE_BASEDIR"
 123     df --inodes "$WEBSITE_CACHE_BASEDIR"
 124
 125     website-cache gc
 126
 127     echo "Website cache filesystem usage after garbage collection:"
 128     df "$WEBSITE_CACHE_BASEDIR"
 129     df --inodes "$WEBSITE_CACHE_BASEDIR"
 130
 131     WEBSITE_CACHE_KEY=$(website-cache key)
 132     if ! website-cache get "$WEBSITE_CACHE_KEY"; then
 133         ./build-website
 134         WEBSITE_BUILD_DIR="config/chroot_local-includes/usr/share/doc/tails/website"
 135         echo "Website cache entry size: $(du --apparent-size --summarize "${WEBSITE_BUILD_DIR}" | cut -f1)"
 136         echo "Website cache entry inodes: $(du --inodes --summarize "${WEBSITE_BUILD_DIR}" | cut -f1)"
 137
 138         if ! website-cache put "$WEBSITE_CACHE_KEY"; then
 139             # Delete incomplete cache entry
 140             rm -rf "${WEBSITE_CACHE_BASEDIR:?}/${WEBSITE_CACHE_KEY:?}"
 141             cat <<EOF
 142 If this has started to fail with "No space left on device",
 143 check whether we're short on actual disk space or on inodes. And then:
 144
 145  - If we're short on actual disk space: have "website-cache gc" ensure
 146    that 10% disk space is available, by deleting as many cache entries
 147    as necessary, by decreasing age.
 148
 149  - Else, if we're short on inodes: either tune the website cache filesystem's
 150    inodes setup to match its actual (which is to store tons of small files),
 151    or migrate to storing cache entries as tarballs.
 152
 153 For more context, see #20150.
 154 EOF
 155             exit 1
 156         fi
 157         echo "Website cache filesystem usage after storing new cache entry:"
 158         df "$WEBSITE_CACHE_BASEDIR"
 159         df --inodes "$WEBSITE_CACHE_BASEDIR"
 160     fi
 161 else
 162     ./build-website
 163 fi
 164
 165 # refresh translations of our programs
 166 ./refresh-translations || fatal "refresh-translations failed ($?)."
 167
 168 # generate list of supported languages
 169 generate-languages-list || fatal "generate-languages-list failed ($?)."
 170
 171 BUILD_ISO_FILENAME="${BUILD_BASENAME}.iso"
 172 BUILD_MANIFEST="${BUILD_BASENAME}.build-manifest"
 173 BUILD_APT_SOURCES="${BUILD_BASENAME}.apt-sources"
 174 BUILD_PACKAGES="${BUILD_BASENAME}.packages"
 175 BUILD_USB_IMAGE_FILENAME="${BUILD_BASENAME}.img"
 176
 177 (
 178     echo "Mirrors:"
 179     apt-mirror debian
 180     apt-mirror debian-security
 181     apt-mirror torproject
 182     echo "Additional sources:"
 183     cat config/chroot_sources/*.chroot
 184 ) >"$BUILD_APT_SOURCES"
 185
 186 # make submodules available in the chroot:
 187 SUBMODULES_SRC="submodules/tails-workarounds"
 188 SUBMODULES_DST="config/chroot_local-includes/tmp/submodules"
 189 mkdir -p "$SUBMODULES_DST"
 190 # shellcheck disable=SC2086
 191 cp -a $SUBMODULES_SRC "$SUBMODULES_DST"/
 192
 193 # Inject the Tor Browser AppArmor policy into the chroot
 194 APPARMOR_D="config/chroot_local-includes/etc/apparmor.d"
 195 cp submodules/torbrowser-launcher/apparmor/torbrowser.Browser.firefox \
 196     "$APPARMOR_D"/
 197 cp submodules/torbrowser-launcher/apparmor/tunables/* \
 198     "$APPARMOR_D"/tunables/
 199
 200 echo "I: Building ISO image ${BUILD_ISO_FILENAME}..."
 201 time lb build noauto "${@}"
 202 BUILD_EXIT_CODE=$?
 203 [ -e binary.iso ] || fatal "lb build failed (${BUILD_EXIT_CODE})."
 204
 205 echo "I: ISO image was successfully created"
 206 print_iso_size binary.iso
 207
 208 echo "I: Hybriding it..."
 209 touch chroot/binary.iso
 210 mount --bind binary.iso chroot/binary.iso
 211 # shellcheck disable=SC2086
 212 chroot chroot isohybrid $ISOHYBRID_OPTS binary.iso || fatal "isohybrid failed"
 213 umount chroot/binary.iso
 214 print_iso_size binary.iso
 215 truncate -s %2048 binary.iso
 216 print_iso_size binary.iso
 217
 218 echo "I: Renaming generated files..."
 219 mv -i binary.iso "${BUILD_ISO_FILENAME}"
 220 mv -i binary.packages "${BUILD_PACKAGES}"
 221
 222 echo "I: Generating build manifest..."
 223 generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
 224
 225 echo "I: Creating USB image ${BUILD_USB_IMAGE_FILENAME}..."
 226 create-usb-image-from-iso "${BUILD_ISO_FILENAME}"