run_test_suite

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -o pipefail
   6
   7 NAME=$(basename "${0}")
   8
   9 GENERAL_DEPENDENCIES="
  10 cucumber
  11 devscripts
  12 dnsmasq-base
  13 ffmpeg
  14 gawk
  15 git
  16 imagemagick
  17 libcap2-bin
  18 libvirt-clients
  19 libvirt-daemon-system
  20 libvirt-dev
  21 libvirt0
  22 obfs4proxy
  23 openssh-server
  24 ovmf
  25 pry
  26 python3-opencv
  27 python3-pil
  28 python3-slixmpp
  29 qemu-system-x86
  30 qrencode
  31 ruby-bindex
  32 ruby-binding-of-caller
  33 ruby-guestfs
  34 ruby-json
  35 ruby-libvirt
  36 ruby-net-dns
  37 ruby-packetfu
  38 ruby-rb-inotify
  39 ruby-rspec
  40 ruby-test-unit
  41 seabios
  42 tcpdump
  43 tcplay
  44 tor
  45 unclutter
  46 virt-viewer
  47 xdotool
  48 xvfb
  49 "
  50
  51 # Recent tor client require a Chutney network with tor >= 0.4.4.6 (#18190)
  52 MINIMUM_TOR_VERSION=0.4.4.6
  53
  54 usage() {
  55     echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
  56 Sets up an appropriate environment and invokes cucumber. Note that this script
  57 must be run from the Tails source directory root.
  58
  59 Options for '@product' features:
  60   --allow-non-root   Normally the test suite must be run as root, but if you
  61                      really know what you are doing this option allows any
  62                      user to run it.
  63   --artifacts-base-uri URI
  64                      Pretend that the artifact is located at URI when printing
  65                      its location during a scenario failure. This is useful if
  66                      you intend to serve the artifacts via the web, for
  67                      instance.
  68   --capture          Captures failed scenarios into videos stored in the
  69                      temporary directory (see --tmpdir below) using x264
  70                      encoding. Requires x264.
  71   --capture-all      Keep videos for all scenarios, including those that
  72                      succeed (implies --capture).
  73   --interactive-debugging
  74                      On failure, pause test suite until pressing Enter. Also
  75                      offer the option to open an interactive Ruby shell (pry)
  76                      in the Cucumber world's context.
  77   --image-bumping-mode
  78                      When any image matching fails, enter an interactive mode
  79                      that allows to update the image. If run from a graphical
  80                      environment, any found candidate image will be displayed
  81                      in a pop-up.
  82   --keep-chutney     Don't ever clean Chutney data directory.
  83                      This can be a big time saver when debugging steps
  84                      when --keep-snapshots is not an option.
  85   --keep-snapshots   Don't ever delete any snapshots (including ones marked as
  86                      temporary). This can be a big time saver when debugging new
  87                      features. Implies --keep-chutney.
  88   --disable-chutney  EXPERIMENTAL: All tests will be run using the real Tor
  89                      network, not a simulated one.
  90                      Expect this to break many test cases.
  91   --late-patch
  92   --late-patch=FILE
  93                      Copies files into the VM before running the test suite,
  94                      which often can avoid a rebuild.
  95                      FILE is a text file where each line contains tab-separated
  96                      source and destination. If only a source is given, it is
  97                      assumed to be a file inside config/chroot_local-includes,
  98                      and the corresponding destination is inferred. Lines with
  99                      a leading \"#\" are ignored.
 100                      FILE is optional, and if not given we copy in all files in
 101                      config/chroot_local-includes that have changed since the
 102                      commit the system under testing was built from, including
 103                      untracked files.
 104   --early-patch      Boots the system with the \"early_patch=umount\" cmdline option,
 105                      so all changes since the commit the system under testing was
 106                      built from are copied in, including untracked, unstaged, and
 107                      uncommitted changes.
 108                      See wiki/src/contribute/build/early-patch.mdwn for details.
 109   --all-tests        Don't skip tests which have a @fragile or @skip_by_default tag.
 110   --tmpdir           Directory where various temporary files are written
 111                      during a test, e.g. VM snapshots and memory dumps,
 112                      failure screenshots, pcap files and disk images
 113                      (default is TMPDIR in the environment, and if unset,
 114                      /tmp/TailsToaster).
 115   --view             Shows the test session in a windows. Requires x11vnc
 116                      and tigervnc-viewer.
 117   --view-interact    The test session can be \"touched\". For debugging purposes
 118                      only.
 119   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
 120   --iso IMAGE        Test '@product' features using IMAGE.
 121   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
 122                      an older version of Tails, which this options sets to
 123                      IMAGE. If none is given, it defaults to the same IMAGE
 124                      given by --iso, which will be good enough for most testing
 125                      purposes.
 126
 127 Note that '@source' features has no relevant options.
 128
 129 CUCUMBER_ARGS can be used to specify which features to be run, but also any
 130 cucumber option, although then you must pass \`--\` first to let this wrapper
 131 script know that we're done with *its* options. For debugging purposes, a
 132 'debug' formatter has been added so pretty debugging can be enabled with
 133 \`--format debug\`. You could even combine the default (pretty) formatter with
 134 pretty debugging printed to a file with \`--format pretty --format debug
 135 --out debug.log\`.
 136 "
 137 }
 138
 139 error() {
 140     echo "${NAME}: error: ${*}" >&2
 141     exit 1
 142 }
 143
 144 package_installed() {
 145     local ret
 146     set +o pipefail
 147     if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
 148         ret=0
 149     else
 150         ret=1
 151     fi
 152     set -o pipefail
 153     return ${ret}
 154 }
 155
 156 check_dependencies() {
 157     while [ -n "${1:-}" ]; do
 158         if ! command -v "${1}" >/dev/null && ! package_installed "${1}" ; then
 159             error "'${1}' is missing, please install it and run again."
 160         fi
 161         shift
 162     done
 163 }
 164
 165 check_tor_version() {
 166     local tor_version
 167     tor_version=$(tor --version | grep -Po '^Tor version \K.*' | sed --regexp-extended 's,[.]$,,')
 168     echo "tor version: $tor_version"
 169     if dpkg --compare-versions "$tor_version" lt "$MINIMUM_TOR_VERSION"; then
 170         error "Please upgrade to tor ${MINIMUM_TOR_VERSION} or newer."
 171     fi
 172 }
 173
 174 check_virt_viewer_version() {
 175     local version
 176     version="$(dpkg-query --show --showformat='${VERSION}' virt-viewer)"
 177     if dpkg --compare-versions "${version}" ge 10.0; then
 178         if ! echo "${version}" | grep -q 'tails$'; then
 179             error 'Please install virt-viewer from the isotester-bookworm APT suite' \
 180                   '(instructions: https://tails.net/contribute/release_process/test/setup/' \
 181                   'details: https://gitlab.tails.boum.org/tails/tails/-/issues/19064)'
 182         fi
 183     fi
 184 }
 185
 186 display_in_use() {
 187     [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
 188 }
 189
 190 next_free_display() {
 191     display_nr=0
 192     while display_in_use ":${display_nr}"; do
 193         display_nr=$((display_nr+1))
 194     done
 195     echo ":${display_nr}"
 196 }
 197
 198 test_suite_cleanup() {
 199     if [ -n "${XVFB_PID:-}" ]; then
 200         (kill -0 "${XVFB_PID}" 2>/dev/null && kill "${XVFB_PID}") || /bin/true
 201     fi
 202     return $?
 203 }
 204
 205 start_xvfb() {
 206     Xvfb "$TARGET_DISPLAY" -screen 0 1024x768x24+32 -noreset >/dev/null 2>&1 &
 207     XVFB_PID=$!
 208     # Wait for Xvfb to run on TARGET_DISPLAY
 209     until display_in_use "$TARGET_DISPLAY"; do
 210         sleep 1
 211     done
 212     echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
 213     # Hide the mouse cursor so it won't be in the way when we are
 214     # trying to match images.
 215     unclutter -display "$TARGET_DISPLAY" -root -idle 0.1 >/dev/null 2>&1 &
 216 }
 217
 218 start_vnc_server() {
 219     check_dependencies x11vnc
 220     # shellcheck disable=SC2086
 221     VNC_SERVER_PORT="$(env -u WAYLAND_DISPLAY x11vnc \
 222                               -listen localhost -display "${TARGET_DISPLAY}" \
 223                               -bg -nopw -forever ${TAILS_X11VNC_OPTS:-} 2>&1 | \
 224                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
 225     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 226 }
 227
 228 start_vnc_viewer() {
 229     opts=
 230     if [[ "${VNC_VIEWER_VIEWONLY}" = yes ]]; then
 231         opts=-viewonly
 232     fi
 233     check_dependencies tigervnc-viewer
 234     xtigervncviewer \
 235         -nojpeg $opts \
 236         -RemoteResize=0 \
 237         -AcceptClipboard=0 \
 238         -SendClipboard=0 \
 239         -SetPrimary=0 \
 240         -SendPrimary=0 \
 241         "localhost:${VNC_SERVER_PORT}" 1>/dev/null 2>&1 &
 242 }
 243
 244 # main script
 245
 246 # Set default values of environment variables used by this script to
 247 # pass options to cucumber (unless they are already set).
 248 ALLOW_NON_ROOT=${ALLOW_NON_ROOT-}
 249 ARTIFACTS_BASE_URI=${ARTIFACTS_BASE_URI-}
 250 CAPTURE=${CAPTURE-}
 251 CAPTURE_ALL=${CAPTURE_ALL-}
 252 ALWAYS_SAVE_JOURNAL=${ALWAYS_SAVE_JOURNAL-}
 253 VNC_VIEWER=${VNC_VIEWER-}
 254 VNC_VIEWER_VIEWONLY=${VNC_VIEWER_VIEWONLY-yes}
 255 VNC_SERVER=${VNC_SERVER-}
 256 INTERACTIVE_DEBUGGING=${INTERACTIVE_DEBUGGING-}
 257 IMAGE_BUMPING_MODE=${IMAGE_BUMPING_MODE-}
 258 KEEP_CHUTNEY=${KEEP_CHUTNEY-}
 259 KEEP_SNAPSHOTS=${KEEP_SNAPSHOTS-}
 260 TAILS_ISO=${TAILS_ISO-}
 261 OLD_TAILS_ISO=${OLD_TAILS_ISO-}
 262 EARLY_PATCH=${EARLY_PATCH-}
 263 EXTRA_BOOT_OPTIONS=${EXTRA_BOOT_OPTIONS-}
 264 ALL_TESTS=${ALL_TESTS-}
 265
 266 LONGOPTS="allow-non-root,artifacts-base-uri:,view,view-interact,vnc-server-only,capture,capture-all,help,tmpdir:,keep-chutney,keep-snapshots,disable-chutney,late-patch::,early-patch,extra-boot-options:,all-tests,iso:,old-iso:,interactive-debugging,image-bumping-mode"
 267 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 268 eval set -- "$OPTS"
 269 while [ $# -gt 0 ]; do
 270     case $1 in
 271         --allow-non-root)
 272             if ! /sbin/getcap /usr/bin/tcpdump | grep -q 'cap_net_raw=eip'; then
 273                 error "/usr/bin/tcpdump lacks cap_net_raw=eip"
 274             fi
 275             ALLOW_NON_ROOT="yes"
 276             ;;
 277         --artifacts-base-uri)
 278             shift
 279             export ARTIFACTS_BASE_URI="${1}"
 280             ;;
 281         --view)
 282             VNC_VIEWER=yes
 283             VNC_VIEWER_VIEWONLY=yes
 284             VNC_SERVER=yes
 285             ;;
 286         --view-interact)
 287             VNC_VIEWER=yes
 288             VNC_VIEWER_VIEWONLY=
 289             VNC_SERVER=yes
 290             ;;
 291         --vnc-server-only)
 292             VNC_VIEWER=
 293             VNC_SERVER=yes
 294             ;;
 295         --capture)
 296             check_dependencies x264
 297             export CAPTURE="yes"
 298             ;;
 299         --capture-all)
 300             check_dependencies x264
 301             export CAPTURE="yes"
 302             export CAPTURE_ALL="yes"
 303             ;;
 304         --interactive-debugging)
 305             export INTERACTIVE_DEBUGGING="yes"
 306             ;;
 307         --image-bumping-mode)
 308             export IMAGE_BUMPING_MODE="yes"
 309             ;;
 310         --keep-chutney)
 311             export KEEP_CHUTNEY="yes"
 312             ;;
 313         --keep-snapshots)
 314             export KEEP_CHUTNEY="yes"
 315             export KEEP_SNAPSHOTS="yes"
 316             ;;
 317         --disable-chutney)
 318             export DISABLE_CHUTNEY="yes"
 319             ;;
 320         --late-patch)
 321             shift
 322             if [ -n "${1:-}" ]; then
 323                 LATE_PATCH="$(realpath -s "$1")"
 324                 if [ ! -e "$LATE_PATCH" ]; then
 325                     error "--late-patch: FILE doesn't exist"
 326                 fi
 327             else
 328                 LATE_PATCH=''
 329             fi
 330             export LATE_PATCH
 331             ;;
 332         --early-patch)
 333             export EARLY_PATCH="yes"
 334             ;;
 335         --extra-boot-options)
 336             shift
 337             export EXTRA_BOOT_OPTIONS="$1"
 338             ;;
 339         --all-tests)
 340             ALL_TESTS="yes"
 341             ;;
 342         --tmpdir)
 343             shift
 344             TMPDIR="$(readlink -f "$1")"
 345             export TMPDIR
 346             ;;
 347         --iso)
 348             shift
 349             TAILS_ISO="$(realpath -s "$1")"
 350             export TAILS_ISO
 351             ;;
 352         --old-iso)
 353             shift
 354             OLD_TAILS_ISO="$(realpath -s "$1")"
 355             export OLD_TAILS_ISO
 356             ;;
 357         --help)
 358             usage
 359             exit 0
 360             ;;
 361         --)
 362             shift
 363             break
 364             ;;
 365     esac
 366     shift
 367 done
 368
 369 trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
 370
 371 if [ "${EUID}" -ne 0 ] && [ -z "${ALLOW_NON_ROOT}" ]; then
 372     error "you are not running as root; if you really know what you are" \
 373           "doing, see the --allow-non-root option"
 374 fi
 375
 376 # shellcheck disable=SC2086
 377 check_dependencies ${GENERAL_DEPENDENCIES}
 378 check_tor_version
 379 check_virt_viewer_version
 380
 381 TARGET_DISPLAY=$(next_free_display)
 382
 383 start_xvfb
 384
 385 if [ -n "${VNC_SERVER:-}" ]; then
 386     start_vnc_server
 387 fi
 388 if [ -n "${VNC_VIEWER:-}" ]; then
 389     start_vnc_viewer
 390 fi
 391
 392 TAGS_ARGS=""
 393 if [ -n "${JENKINS_URL:-}" ]; then
 394     . auto/scripts/utils.sh
 395
 396     if echo "${GIT_BRANCH}" | grep -q -E '[+-]real-Tor$'; then
 397         export DISABLE_CHUTNEY="yes"
 398         TAGS_ARGS="--tag @supports_real_tor"
 399     # The current Git state may not reflect the state at the time the
 400     # upstream job was started (e.g. since then we git fetch + git
 401     # reset --hard) so we trust the Git state described in Jenkins'
 402     # environment variables instead.
 403     elif echo "${GIT_BRANCH}" | grep -q -E '[+-]force-all-tests$' \
 404        || git_on_a_tag \
 405        || [ "${GIT_BRANCH#origin/}" = feature/bookworm ] \
 406        || [ "${GIT_BRANCH#origin/}" = testing ] \
 407        || [ "${GIT_BRANCH#origin/}" = devel ] \
 408        || [ -n "${ALL_TESTS:-}" ]; then
 409         TAGS_ARGS=""
 410     else
 411         TAGS_ARGS="${TAGS_ARGS} --tag ~@fragile --tag ~@skip_by_default"
 412     fi
 413     if [ "${UPSTREAMJOB_GIT_COMMIT}" != "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}" ] && \
 414            git_only_doc_changes_since "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}"; then
 415         TAGS_ARGS="${TAGS_ARGS} --tag @doc"
 416     fi
 417 fi
 418
 419 export USER_DISPLAY="${DISPLAY:-}"
 420 export DISPLAY=${TARGET_DISPLAY}
 421
 422 # shellcheck disable=SC2086
 423 cucumber --expand ${TAGS_ARGS} "${@}"