run_test_suite

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -o pipefail
   6
   7 NAME=$(basename "${0}")
   8
   9 GENERAL_DEPENDENCIES="
  10 cucumber
  11 devscripts
  12 dnsmasq-base
  13 ffmpeg
  14 gawk
  15 git
  16 i18nspector
  17 imagemagick
  18 libcap2-bin
  19 libvirt-clients
  20 libvirt-daemon-system
  21 libvirt-dev
  22 libvirt0
  23 obfs4proxy
  24 openssh-server
  25 ovmf
  26 pry
  27 python3-opencv
  28 python3-pil
  29 python3-potr
  30 python3-slixmpp
  31 qemu-system-x86
  32 redir
  33 ruby-guestfs
  34 ruby-json
  35 ruby-libvirt
  36 ruby-packetfu
  37 ruby-rb-inotify
  38 ruby-rspec
  39 ruby-test-unit
  40 seabios
  41 tcpdump
  42 tcplay
  43 tor
  44 unclutter
  45 virt-viewer
  46 xdotool
  47 xvfb
  48 "
  49
  50 usage() {
  51     echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
  52 Sets up an appropriate environment and invokes cucumber. Note that this script
  53 must be run from the Tails source directory root.
  54
  55 Options for '@product' features:
  56   --allow-non-root   Normally the test suite must be run as root, but if you
  57                      really know what you are doing this option allows any
  58                      user to run it.
  59   --artifacts-base-uri URI
  60                      Pretend that the artifact is located at URI when printing
  61                      its location during a scenario failure. This is useful if
  62                      you intend to serve the artifacts via the web, for
  63                      instance.
  64   --capture          Captures failed scenarios into videos stored in the
  65                      temporary directory (see --tmpdir below) using x264
  66                      encoding. Requires x264.
  67   --capture-all      Keep videos for all scenarios, including those that
  68                      succeed (implies --capture).
  69   --interactive-debugging
  70                      On failure, pause test suite until pressing Enter. Also
  71                      offer the option to open an interactive Ruby shell (pry)
  72                      in the Cucumber world's context.
  73   --image-bumping-mode
  74                      When any image matching fails, enter an interactive mode
  75                      that allows to update the image. If run from a graphical
  76                      environment, any found candidate image will be displayed
  77                      in a pop-up.
  78   --keep-chutney     Don't ever clean Chutney data directory.
  79                      This can be a big time saver when debugging steps
  80                      when --keep-snapshots is not an option.
  81   --keep-snapshots   Don't ever delete any snapshots (including ones marked as
  82                      temporary). This can be a big time saver when debugging new
  83                      features. Implies --keep-chutney.
  84   --tmpdir           Directory where various temporary files are written
  85                      during a test, e.g. VM snapshots and memory dumps,
  86                      failure screenshots, pcap files and disk images
  87                      (default is TMPDIR in the environment, and if unset,
  88                      /tmp/TailsToaster).
  89   --view             Shows the test session in a windows. Requires x11vnc
  90                      and tigervnc-viewer.
  91   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
  92   --iso IMAGE        Test '@product' features using IMAGE.
  93   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
  94                      an older version of Tails, which this options sets to
  95                      IMAGE. If none is given, it defaults to the same IMAGE
  96                      given by --iso, which will be good enough for most testing
  97                      purposes.
  98
  99 Note that '@source' features has no relevant options.
 100
 101 CUCUMBER_ARGS can be used to specify which features to be run, but also any
 102 cucumber option, although then you must pass \`--\` first to let this wrapper
 103 script know that we're done with *its* options. For debugging purposes, a
 104 'debug' formatter has been added so pretty debugging can be enabled with
 105 \`--format debug\`. You could even combine the default (pretty) formatter with
 106 pretty debugging printed to a file with \`--format pretty --format debug
 107 --out debug.log\`.
 108 "
 109 }
 110
 111 error() {
 112     echo "${NAME}: error: ${*}" >&2
 113     exit 1
 114 }
 115
 116 package_installed() {
 117     local ret
 118     set +o pipefail
 119     if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
 120         ret=0
 121     else
 122         ret=1
 123     fi
 124     set -o pipefail
 125     return ${ret}
 126 }
 127
 128 check_dependencies() {
 129     while [ -n "${1:-}" ]; do
 130         if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
 131             error "'${1}' is missing, please install it and run again."
 132         fi
 133         shift
 134     done
 135 }
 136
 137 display_in_use() {
 138     [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
 139 }
 140
 141 next_free_display() {
 142     display_nr=0
 143     while display_in_use ":${display_nr}"; do
 144         display_nr=$((display_nr+1))
 145     done
 146     echo ":${display_nr}"
 147 }
 148
 149 test_suite_cleanup() {
 150     if [ -n "${XVFB_PID:-}" ]; then
 151         (kill -0 "${XVFB_PID}" 2>/dev/null && kill "${XVFB_PID}") || /bin/true
 152     fi
 153     return $?
 154 }
 155
 156 start_xvfb() {
 157     Xvfb "$TARGET_DISPLAY" -screen 0 1024x768x24+32 >/dev/null 2>&1 &
 158     XVFB_PID=$!
 159     # Wait for Xvfb to run on TARGET_DISPLAY
 160     until display_in_use "$TARGET_DISPLAY"; do
 161         sleep 1
 162     done
 163     echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
 164     # Hide the mouse cursor so it won't be in the way when we are
 165     # trying to match images.
 166     unclutter -display "$TARGET_DISPLAY" -root -idle 0.1 >/dev/null 2>&1 &
 167 }
 168
 169 start_vnc_server() {
 170     check_dependencies x11vnc
 171     VNC_SERVER_PORT="$(x11vnc -listen localhost -display "${TARGET_DISPLAY}" \
 172                               -bg -nopw -forever 2>&1 | \
 173                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
 174     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 175 }
 176
 177 start_vnc_viewer() {
 178     check_dependencies tigervnc-viewer
 179     xtigervncviewer \
 180         -nojpeg -viewonly \
 181         -RemoteResize=0 \
 182         -AcceptClipboard=0 \
 183         -SendClipboard=0 \
 184         -SetPrimary=0 \
 185         -SendPrimary=0 \
 186         "localhost:${VNC_SERVER_PORT}" 1>/dev/null 2>&1 &
 187 }
 188
 189 # main script
 190
 191 # Unset all environment variables used by this script to pass options
 192 # to cucumber, except TMPDIR since we explicitly want to support
 193 # setting it that way.
 194 ALLOW_NON_ROOT=
 195 ARTIFACTS_BASE_URI=
 196 CAPTURE=
 197 CAPTURE_ALL=
 198 VNC_VIEWER=
 199 VNC_SERVER=
 200 INTERACTIVE_DEBUGGING=
 201 IMAGE_BUMPING_MODE=
 202 KEEP_CHUTNEY=
 203 KEEP_SNAPSHOTS=
 204 TAILS_ISO=
 205 OLD_TAILS_ISO=
 206
 207 LONGOPTS="allow-non-root,artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-chutney,keep-snapshots,iso:,old-iso:,interactive-debugging,image-bumping-mode"
 208 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 209 eval set -- "$OPTS"
 210 while [ $# -gt 0 ]; do
 211     case $1 in
 212         --allow-non-root)
 213             ALLOW_NON_ROOT="yes"
 214             ;;
 215         --artifacts-base-uri)
 216             shift
 217             export ARTIFACTS_BASE_URI="${1}"
 218             ;;
 219         --view)
 220             VNC_VIEWER=yes
 221             VNC_SERVER=yes
 222             ;;
 223         --vnc-server-only)
 224             VNC_VIEWER=
 225             VNC_SERVER=yes
 226             ;;
 227         --capture)
 228             check_dependencies x264
 229             export CAPTURE="yes"
 230             ;;
 231         --capture-all)
 232             check_dependencies x264
 233             export CAPTURE="yes"
 234             export CAPTURE_ALL="yes"
 235             ;;
 236         --interactive-debugging)
 237             export INTERACTIVE_DEBUGGING="yes"
 238             ;;
 239         --image-bumping-mode)
 240             export IMAGE_BUMPING_MODE="yes"
 241             ;;
 242         --keep-chutney)
 243             export KEEP_CHUTNEY="yes"
 244             ;;
 245         --keep-snapshots)
 246             export KEEP_CHUTNEY="yes"
 247             export KEEP_SNAPSHOTS="yes"
 248             ;;
 249         --tmpdir)
 250             shift
 251             export TMPDIR="$(readlink -f "$1")"
 252             ;;
 253         --iso)
 254             shift
 255             export TAILS_ISO="$(realpath -s "$1")"
 256             ;;
 257         --old-iso)
 258             shift
 259             export OLD_TAILS_ISO="$(realpath -s "$1")"
 260             ;;
 261         --help)
 262             usage
 263             exit 0
 264             ;;
 265         --)
 266             shift
 267             break
 268             ;;
 269     esac
 270     shift
 271 done
 272
 273 trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
 274
 275 if [ "${EUID}" -ne 0 ] && [ -z "${ALLOW_NON_ROOT}" ]; then
 276     error "you are not running as root; if you really know what you are" \
 277           "doing, see the --allow-non-root option"
 278 fi
 279
 280 check_dependencies ${GENERAL_DEPENDENCIES}
 281
 282 TARGET_DISPLAY=$(next_free_display)
 283
 284 start_xvfb
 285
 286 if [ -n "${VNC_SERVER:-}" ]; then
 287     start_vnc_server
 288 fi
 289 if [ -n "${VNC_VIEWER:-}" ]; then
 290     start_vnc_viewer
 291 fi
 292
 293 TAGS_ARGS=""
 294 if [ -n "${JENKINS_URL:-}" ]; then
 295     # The current Git state may not reflect the state at the time the
 296     # upstream job was started (e.g. since then we git fetch + git
 297     # reset --hard) so we trust the Git state described in Jenkins'
 298     # environment variables instead.
 299     . auto/scripts/utils.sh
 300     if echo "${GIT_BRANCH}" | grep -q -E '[+-]force-all-tests$' \
 301        || git_on_a_tag \
 302        || [ "${GIT_BRANCH#origin/}" = testing ] \
 303        || [ "${GIT_BRANCH#origin/}" = feature/tor-nightly-master ] \
 304        || [ "${GIT_BRANCH#origin/}" = devel ] ; then
 305         TAGS_ARGS=""
 306     else
 307         TAGS_ARGS="--tag ~@fragile"
 308     fi
 309     if [ "${UPSTREAMJOB_GIT_COMMIT}" != "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}" ] && \
 310            git_only_doc_changes_since "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}"; then
 311         TAGS_ARGS="${TAGS_ARGS} --tag @doc"
 312     fi
 313 fi
 314
 315 export USER_DISPLAY="${DISPLAY:-}"
 316 export DISPLAY=${TARGET_DISPLAY}
 317
 318 cucumber --expand ${TAGS_ARGS} "${@}"