tests/vm/basevm.py

   1 #
   2 # VM testing base class
   3 #
   4 # Copyright 2017-2019 Red Hat Inc.
   5 #
   6 # Authors:
   7 #  Fam Zheng <famz@redhat.com>
   8 #  Gerd Hoffmann <kraxel@redhat.com>
   9 #
  10 # This code is licensed under the GPL version 2 or later.  See
  11 # the COPYING file in the top-level directory.
  12 #
  13
  14 import os
  15 import re
  16 import sys
  17 import socket
  18 import logging
  19 import time
  20 import datetime
  21 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
  22 from qemu.accel import kvm_available
  23 from qemu.machine import QEMUMachine
  24 import subprocess
  25 import hashlib
  26 import argparse
  27 import atexit
  28 import tempfile
  29 import shutil
  30 import multiprocessing
  31 import traceback
  32 import shlex
  33
  34 SSH_KEY_FILE = os.path.join(os.path.dirname(__file__),
  35                "..", "keys", "id_rsa")
  36 SSH_PUB_KEY_FILE = os.path.join(os.path.dirname(__file__),
  37                    "..", "keys", "id_rsa.pub")
  38
  39 # This is the standard configuration.
  40 # Any or all of these can be overridden by
  41 # passing in a config argument to the VM constructor.
  42 DEFAULT_CONFIG = {
  43     'cpu'             : "max",
  44     'machine'         : 'pc',
  45     'guest_user'      : "qemu",
  46     'guest_pass'      : "qemupass",
  47     'root_pass'       : "qemupass",
  48     'ssh_key_file'    : SSH_KEY_FILE,
  49     'ssh_pub_key_file': SSH_PUB_KEY_FILE,
  50     'memory'          : "4G",
  51     'extra_args'      : [],
  52     'qemu_args'       : "",
  53     'dns'             : "",
  54     'ssh_port'        : 0,
  55     'install_cmds'    : "",
  56     'boot_dev_type'   : "block",
  57     'ssh_timeout'     : 1,
  58 }
  59 BOOT_DEVICE = {
  60     'block' :  "-drive file={},if=none,id=drive0,cache=writeback "\
  61                "-device virtio-blk,drive=drive0,bootindex=0",
  62     'scsi'  :  "-device virtio-scsi-device,id=scsi "\
  63                "-drive file={},format=raw,if=none,id=hd0 "\
  64                "-device scsi-hd,drive=hd0,bootindex=0",
  65 }
  66 class BaseVM(object):
  67
  68     envvars = [
  69         "https_proxy",
  70         "http_proxy",
  71         "ftp_proxy",
  72         "no_proxy",
  73     ]
  74
  75     # The script to run in the guest that builds QEMU
  76     BUILD_SCRIPT = ""
  77     # The guest name, to be overridden by subclasses
  78     name = "#base"
  79     # The guest architecture, to be overridden by subclasses
  80     arch = "#arch"
  81     # command to halt the guest, can be overridden by subclasses
  82     poweroff = "poweroff"
  83     # enable IPv6 networking
  84     ipv6 = True
  85     # This is the timeout on the wait for console bytes.
  86     socket_timeout = 120
  87     # Scale up some timeouts under TCG.
  88     # 4 is arbitrary, but greater than 2,
  89     # since we found we need to wait more than twice as long.
  90     tcg_ssh_timeout_multiplier = 4
  91     def __init__(self, args, config=None):
  92         self._guest = None
  93         self._genisoimage = args.genisoimage
  94         self._build_path = args.build_path
  95         self._efi_aarch64 = args.efi_aarch64
  96         # Allow input config to override defaults.
  97         self._config = DEFAULT_CONFIG.copy()
  98         if config != None:
  99             self._config.update(config)
 100         self.validate_ssh_keys()
 101         self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-",
 102                                                          suffix=".tmp",
 103                                                          dir="."))
 104         atexit.register(shutil.rmtree, self._tmpdir)
 105         # Copy the key files to a temporary directory.
 106         # Also chmod the key file to agree with ssh requirements.
 107         self._config['ssh_key'] = \
 108             open(self._config['ssh_key_file']).read().rstrip()
 109         self._config['ssh_pub_key'] = \
 110             open(self._config['ssh_pub_key_file']).read().rstrip()
 111         self._ssh_tmp_key_file = os.path.join(self._tmpdir, "id_rsa")
 112         open(self._ssh_tmp_key_file, "w").write(self._config['ssh_key'])
 113         subprocess.check_call(["chmod", "600", self._ssh_tmp_key_file])
 114
 115         self._ssh_tmp_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub")
 116         open(self._ssh_tmp_pub_key_file,
 117              "w").write(self._config['ssh_pub_key'])
 118
 119         self.debug = args.debug
 120         self._console_log_path = None
 121         if args.log_console:
 122                 self._console_log_path = \
 123                          os.path.join(os.path.expanduser("~/.cache/qemu-vm"),
 124                                       "{}.install.log".format(self.name))
 125         self._stderr = sys.stderr
 126         self._devnull = open(os.devnull, "w")
 127         if self.debug:
 128             self._stdout = sys.stdout
 129         else:
 130             self._stdout = self._devnull
 131         netdev = "user,id=vnet,hostfwd=:127.0.0.1:{}-:22"
 132         self._args = [ \
 133             "-nodefaults", "-m", self._config['memory'],
 134             "-cpu", self._config['cpu'],
 135             "-netdev",
 136             netdev.format(self._config['ssh_port']) +
 137             (",ipv6=no" if not self.ipv6 else "") +
 138             (",dns=" + self._config['dns'] if self._config['dns'] else ""),
 139             "-device", "virtio-net-pci,netdev=vnet",
 140             "-vnc", "127.0.0.1:0,to=20"]
 141         if args.jobs and args.jobs > 1:
 142             self._args += ["-smp", "%d" % args.jobs]
 143         if kvm_available(self.arch):
 144             self._args += ["-enable-kvm"]
 145         else:
 146             logging.info("KVM not available, not using -enable-kvm")
 147         self._data_args = []
 148
 149         if self._config['qemu_args'] != None:
 150             qemu_args = self._config['qemu_args']
 151             qemu_args = qemu_args.replace('\n',' ').replace('\r','')
 152             # shlex groups quoted arguments together
 153             # we need this to keep the quoted args together for when
 154             # the QEMU command is issued later.
 155             args = shlex.split(qemu_args)
 156             self._config['extra_args'] = []
 157             for arg in args:
 158                 if arg:
 159                     # Preserve quotes around arguments.
 160                     # shlex above takes them out, so add them in.
 161                     if " " in arg:
 162                         arg = '"{}"'.format(arg)
 163                     self._config['extra_args'].append(arg)
 164
 165     def validate_ssh_keys(self):
 166         """Check to see if the ssh key files exist."""
 167         if 'ssh_key_file' not in self._config or\
 168            not os.path.exists(self._config['ssh_key_file']):
 169             raise Exception("ssh key file not found.")
 170         if 'ssh_pub_key_file' not in self._config or\
 171            not os.path.exists(self._config['ssh_pub_key_file']):
 172                raise Exception("ssh pub key file not found.")
 173
 174     def wait_boot(self, wait_string=None):
 175         """Wait for the standard string we expect
 176            on completion of a normal boot.
 177            The user can also choose to override with an
 178            alternate string to wait for."""
 179         if wait_string is None:
 180             if self.login_prompt is None:
 181                 raise Exception("self.login_prompt not defined")
 182             wait_string = self.login_prompt
 183         # Intentionally bump up the default timeout under TCG,
 184         # since the console wait below takes longer.
 185         timeout = self.socket_timeout
 186         if not kvm_available(self.arch):
 187             timeout *= 8
 188         self.console_init(timeout=timeout)
 189         self.console_wait(wait_string)
 190
 191     def _download_with_cache(self, url, sha256sum=None, sha512sum=None):
 192         def check_sha256sum(fname):
 193             if not sha256sum:
 194                 return True
 195             checksum = subprocess.check_output(["sha256sum", fname]).split()[0]
 196             return sha256sum == checksum.decode("utf-8")
 197
 198         def check_sha512sum(fname):
 199             if not sha512sum:
 200                 return True
 201             checksum = subprocess.check_output(["sha512sum", fname]).split()[0]
 202             return sha512sum == checksum.decode("utf-8")
 203
 204         cache_dir = os.path.expanduser("~/.cache/qemu-vm/download")
 205         if not os.path.exists(cache_dir):
 206             os.makedirs(cache_dir)
 207         fname = os.path.join(cache_dir,
 208                              hashlib.sha1(url.encode("utf-8")).hexdigest())
 209         if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname):
 210             return fname
 211         logging.debug("Downloading %s to %s...", url, fname)
 212         subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"],
 213                               stdout=self._stdout, stderr=self._stderr)
 214         os.rename(fname + ".download", fname)
 215         return fname
 216
 217     def _ssh_do(self, user, cmd, check):
 218         ssh_cmd = ["ssh",
 219                    "-t",
 220                    "-o", "StrictHostKeyChecking=no",
 221                    "-o", "UserKnownHostsFile=" + os.devnull,
 222                    "-o",
 223                    "ConnectTimeout={}".format(self._config["ssh_timeout"]),
 224                    "-p", self.ssh_port, "-i", self._ssh_tmp_key_file]
 225         # If not in debug mode, set ssh to quiet mode to
 226         # avoid printing the results of commands.
 227         if not self.debug:
 228             ssh_cmd.append("-q")
 229         for var in self.envvars:
 230             ssh_cmd += ['-o', "SendEnv=%s" % var ]
 231         assert not isinstance(cmd, str)
 232         ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd)
 233         logging.debug("ssh_cmd: %s", " ".join(ssh_cmd))
 234         r = subprocess.call(ssh_cmd)
 235         if check and r != 0:
 236             raise Exception("SSH command failed: %s" % cmd)
 237         return r
 238
 239     def ssh(self, *cmd):
 240         return self._ssh_do(self._config["guest_user"], cmd, False)
 241
 242     def ssh_root(self, *cmd):
 243         return self._ssh_do("root", cmd, False)
 244
 245     def ssh_check(self, *cmd):
 246         self._ssh_do(self._config["guest_user"], cmd, True)
 247
 248     def ssh_root_check(self, *cmd):
 249         self._ssh_do("root", cmd, True)
 250
 251     def build_image(self, img):
 252         raise NotImplementedError
 253
 254     def exec_qemu_img(self, *args):
 255         cmd = [os.environ.get("QEMU_IMG", "qemu-img")]
 256         cmd.extend(list(args))
 257         subprocess.check_call(cmd)
 258
 259     def add_source_dir(self, src_dir):
 260         name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5]
 261         tarfile = os.path.join(self._tmpdir, name + ".tar")
 262         logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir)
 263         subprocess.check_call(["./scripts/archive-source.sh", tarfile],
 264                               cwd=src_dir, stdin=self._devnull,
 265                               stdout=self._stdout, stderr=self._stderr)
 266         self._data_args += ["-drive",
 267                             "file=%s,if=none,id=%s,cache=writeback,format=raw" % \
 268                                     (tarfile, name),
 269                             "-device",
 270                             "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)]
 271
 272     def boot(self, img, extra_args=[]):
 273         boot_dev = BOOT_DEVICE[self._config['boot_dev_type']]
 274         boot_params = boot_dev.format(img)
 275         args = self._args + boot_params.split(' ')
 276         args += self._data_args + extra_args + self._config['extra_args']
 277         logging.debug("QEMU args: %s", " ".join(args))
 278         qemu_path = get_qemu_path(self.arch, self._build_path)
 279
 280         # Since console_log_path is only set when the user provides the
 281         # log_console option, we will set drain_console=True so the
 282         # console is always drained.
 283         guest = QEMUMachine(binary=qemu_path, args=args,
 284                             console_log=self._console_log_path,
 285                             drain_console=True)
 286         guest.set_machine(self._config['machine'])
 287         guest.set_console()
 288         try:
 289             guest.launch()
 290         except:
 291             logging.error("Failed to launch QEMU, command line:")
 292             logging.error(" ".join([qemu_path] + args))
 293             logging.error("Log:")
 294             logging.error(guest.get_log())
 295             logging.error("QEMU version >= 2.10 is required")
 296             raise
 297         atexit.register(self.shutdown)
 298         self._guest = guest
 299         # Init console so we can start consuming the chars.
 300         self.console_init()
 301         usernet_info = guest.qmp("human-monitor-command",
 302                                  command_line="info usernet")
 303         self.ssh_port = None
 304         for l in usernet_info["return"].splitlines():
 305             fields = l.split()
 306             if "TCP[HOST_FORWARD]" in fields and "22" in fields:
 307                 self.ssh_port = l.split()[3]
 308         if not self.ssh_port:
 309             raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \
 310                             usernet_info)
 311
 312     def console_init(self, timeout = None):
 313         if timeout == None:
 314             timeout = self.socket_timeout
 315         vm = self._guest
 316         vm.console_socket.settimeout(timeout)
 317         self.console_raw_path = os.path.join(vm._temp_dir,
 318                                              vm._name + "-console.raw")
 319         self.console_raw_file = open(self.console_raw_path, 'wb')
 320
 321     def console_log(self, text):
 322         for line in re.split("[\r\n]", text):
 323             # filter out terminal escape sequences
 324             line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line)
 325             line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line)
 326             # replace unprintable chars
 327             line = re.sub("\x1b", "<esc>", line)
 328             line = re.sub("[\x00-\x1f]", ".", line)
 329             line = re.sub("[\x80-\xff]", ".", line)
 330             if line == "":
 331                 continue
 332             # log console line
 333             sys.stderr.write("con recv: %s\n" % line)
 334
 335     def console_wait(self, expect, expectalt = None):
 336         vm = self._guest
 337         output = ""
 338         while True:
 339             try:
 340                 chars = vm.console_socket.recv(1)
 341                 if self.console_raw_file:
 342                     self.console_raw_file.write(chars)
 343                     self.console_raw_file.flush()
 344             except socket.timeout:
 345                 sys.stderr.write("console: *** read timeout ***\n")
 346                 sys.stderr.write("console: waiting for: '%s'\n" % expect)
 347                 if not expectalt is None:
 348                     sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt)
 349                 sys.stderr.write("console: line buffer:\n")
 350                 sys.stderr.write("\n")
 351                 self.console_log(output.rstrip())
 352                 sys.stderr.write("\n")
 353                 raise
 354             output += chars.decode("latin1")
 355             if expect in output:
 356                 break
 357             if not expectalt is None and expectalt in output:
 358                 break
 359             if "\r" in output or "\n" in output:
 360                 lines = re.split("[\r\n]", output)
 361                 output = lines.pop()
 362                 if self.debug:
 363                     self.console_log("\n".join(lines))
 364         if self.debug:
 365             self.console_log(output)
 366         if not expectalt is None and expectalt in output:
 367             return False
 368         return True
 369
 370     def console_consume(self):
 371         vm = self._guest
 372         output = ""
 373         vm.console_socket.setblocking(0)
 374         while True:
 375             try:
 376                 chars = vm.console_socket.recv(1)
 377             except:
 378                 break
 379             output += chars.decode("latin1")
 380             if "\r" in output or "\n" in output:
 381                 lines = re.split("[\r\n]", output)
 382                 output = lines.pop()
 383                 if self.debug:
 384                     self.console_log("\n".join(lines))
 385         if self.debug:
 386             self.console_log(output)
 387         vm.console_socket.setblocking(1)
 388
 389     def console_send(self, command):
 390         vm = self._guest
 391         if self.debug:
 392             logline = re.sub("\n", "<enter>", command)
 393             logline = re.sub("[\x00-\x1f]", ".", logline)
 394             sys.stderr.write("con send: %s\n" % logline)
 395         for char in list(command):
 396             vm.console_socket.send(char.encode("utf-8"))
 397             time.sleep(0.01)
 398
 399     def console_wait_send(self, wait, command):
 400         self.console_wait(wait)
 401         self.console_send(command)
 402
 403     def console_ssh_init(self, prompt, user, pw):
 404         sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" \
 405                      % self._config['ssh_pub_key'].rstrip()
 406         self.console_wait_send("login:",    "%s\n" % user)
 407         self.console_wait_send("Password:", "%s\n" % pw)
 408         self.console_wait_send(prompt,      "mkdir .ssh\n")
 409         self.console_wait_send(prompt,      sshkey_cmd)
 410         self.console_wait_send(prompt,      "chmod 755 .ssh\n")
 411         self.console_wait_send(prompt,      "chmod 644 .ssh/authorized_keys\n")
 412
 413     def console_sshd_config(self, prompt):
 414         self.console_wait(prompt)
 415         self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n")
 416         for var in self.envvars:
 417             self.console_wait(prompt)
 418             self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var)
 419
 420     def print_step(self, text):
 421         sys.stderr.write("### %s ...\n" % text)
 422
 423     def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"):
 424         # Allow more time for VM to boot under TCG.
 425         if not kvm_available(self.arch):
 426             seconds *= self.tcg_ssh_timeout_multiplier
 427         starttime = datetime.datetime.now()
 428         endtime = starttime + datetime.timedelta(seconds=seconds)
 429         cmd_success = False
 430         while datetime.datetime.now() < endtime:
 431             if wait_root and self.ssh_root(cmd) == 0:
 432                 cmd_success = True
 433                 break
 434             elif self.ssh(cmd) == 0:
 435                 cmd_success = True
 436                 break
 437             seconds = (endtime - datetime.datetime.now()).total_seconds()
 438             logging.debug("%ds before timeout", seconds)
 439             time.sleep(1)
 440         if not cmd_success:
 441             raise Exception("Timeout while waiting for guest ssh")
 442
 443     def shutdown(self):
 444         self._guest.shutdown()
 445
 446     def wait(self):
 447         self._guest.wait()
 448
 449     def graceful_shutdown(self):
 450         self.ssh_root(self.poweroff)
 451         self._guest.wait()
 452
 453     def qmp(self, *args, **kwargs):
 454         return self._guest.qmp(*args, **kwargs)
 455
 456     def gen_cloud_init_iso(self):
 457         cidir = self._tmpdir
 458         mdata = open(os.path.join(cidir, "meta-data"), "w")
 459         name = self.name.replace(".","-")
 460         mdata.writelines(["instance-id: {}-vm-0\n".format(name),
 461                           "local-hostname: {}-guest\n".format(name)])
 462         mdata.close()
 463         udata = open(os.path.join(cidir, "user-data"), "w")
 464         print("guest user:pw {}:{}".format(self._config['guest_user'],
 465                                            self._config['guest_pass']))
 466         udata.writelines(["#cloud-config\n",
 467                           "chpasswd:\n",
 468                           "  list: |\n",
 469                           "    root:%s\n" % self._config['root_pass'],
 470                           "    %s:%s\n" % (self._config['guest_user'],
 471                                            self._config['guest_pass']),
 472                           "  expire: False\n",
 473                           "users:\n",
 474                           "  - name: %s\n" % self._config['guest_user'],
 475                           "    sudo: ALL=(ALL) NOPASSWD:ALL\n",
 476                           "    ssh-authorized-keys:\n",
 477                           "    - %s\n" % self._config['ssh_pub_key'],
 478                           "  - name: root\n",
 479                           "    ssh-authorized-keys:\n",
 480                           "    - %s\n" % self._config['ssh_pub_key'],
 481                           "locale: en_US.UTF-8\n"])
 482         proxy = os.environ.get("http_proxy")
 483         if not proxy is None:
 484             udata.writelines(["apt:\n",
 485                               "  proxy: %s" % proxy])
 486         udata.close()
 487         subprocess.check_call([self._genisoimage, "-output", "cloud-init.iso",
 488                                "-volid", "cidata", "-joliet", "-rock",
 489                                "user-data", "meta-data"],
 490                               cwd=cidir,
 491                               stdin=self._devnull, stdout=self._stdout,
 492                               stderr=self._stdout)
 493         return os.path.join(cidir, "cloud-init.iso")
 494
 495 def get_qemu_path(arch, build_path=None):
 496     """Fetch the path to the qemu binary."""
 497     # If QEMU environment variable set, it takes precedence
 498     if "QEMU" in os.environ:
 499         qemu_path = os.environ["QEMU"]
 500     elif build_path:
 501         qemu_path = os.path.join(build_path, arch + "-softmmu")
 502         qemu_path = os.path.join(qemu_path, "qemu-system-" + arch)
 503     else:
 504         # Default is to use system path for qemu.
 505         qemu_path = "qemu-system-" + arch
 506     return qemu_path
 507
 508 def get_qemu_version(qemu_path):
 509     """Get the version number from the current QEMU,
 510        and return the major number."""
 511     output = subprocess.check_output([qemu_path, '--version'])
 512     version_line = output.decode("utf-8")
 513     version_num = re.split(' |\(', version_line)[3].split('.')[0]
 514     return int(version_num)
 515
 516 def parse_config(config, args):
 517     """ Parse yaml config and populate our config structure.
 518         The yaml config allows the user to override the
 519         defaults for VM parameters.  In many cases these
 520         defaults can be overridden without rebuilding the VM."""
 521     if args.config:
 522         config_file = args.config
 523     elif 'QEMU_CONFIG' in os.environ:
 524         config_file = os.environ['QEMU_CONFIG']
 525     else:
 526         return config
 527     if not os.path.exists(config_file):
 528         raise Exception("config file {} does not exist".format(config_file))
 529     # We gracefully handle importing the yaml module
 530     # since it might not be installed.
 531     # If we are here it means the user supplied a .yml file,
 532     # so if the yaml module is not installed we will exit with error.
 533     try:
 534         import yaml
 535     except ImportError:
 536         print("The python3-yaml package is needed "\
 537               "to support config.yaml files")
 538         # Instead of raising an exception we exit to avoid
 539         # a raft of messy (expected) errors to stdout.
 540         exit(1)
 541     with open(config_file) as f:
 542         yaml_dict = yaml.safe_load(f)
 543
 544     if 'qemu-conf' in yaml_dict:
 545         config.update(yaml_dict['qemu-conf'])
 546     else:
 547         raise Exception("config file {} is not valid"\
 548                         " missing qemu-conf".format(config_file))
 549     return config
 550
 551 def parse_args(vmcls):
 552
 553     def get_default_jobs():
 554         if multiprocessing.cpu_count() > 1:
 555             if kvm_available(vmcls.arch):
 556                 return multiprocessing.cpu_count() // 2
 557             elif os.uname().machine == "x86_64" and \
 558                  vmcls.arch in ["aarch64", "x86_64", "i386"]:
 559                 # MTTCG is available on these arches and we can allow
 560                 # more cores. but only up to a reasonable limit. User
 561                 # can always override these limits with --jobs.
 562                 return min(multiprocessing.cpu_count() // 2, 8)
 563         else:
 564             return 1
 565
 566     parser = argparse.ArgumentParser(
 567         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 568         description="Utility for provisioning VMs and running builds",
 569         epilog="""Remaining arguments are passed to the command.
 570         Exit codes: 0 = success, 1 = command line error,
 571         2 = environment initialization failed,
 572         3 = test command failed""")
 573     parser.add_argument("--debug", "-D", action="store_true",
 574                         help="enable debug output")
 575     parser.add_argument("--image", "-i", default="%s.img" % vmcls.name,
 576                         help="image file name")
 577     parser.add_argument("--force", "-f", action="store_true",
 578                         help="force build image even if image exists")
 579     parser.add_argument("--jobs", type=int, default=get_default_jobs(),
 580                         help="number of virtual CPUs")
 581     parser.add_argument("--verbose", "-V", action="store_true",
 582                         help="Pass V=1 to builds within the guest")
 583     parser.add_argument("--build-image", "-b", action="store_true",
 584                         help="build image")
 585     parser.add_argument("--build-qemu",
 586                         help="build QEMU from source in guest")
 587     parser.add_argument("--build-target",
 588                         help="QEMU build target", default="check")
 589     parser.add_argument("--build-path", default=None,
 590                         help="Path of build directory, "\
 591                         "for using build tree QEMU binary. ")
 592     parser.add_argument("--interactive", "-I", action="store_true",
 593                         help="Interactively run command")
 594     parser.add_argument("--snapshot", "-s", action="store_true",
 595                         help="run tests with a snapshot")
 596     parser.add_argument("--genisoimage", default="genisoimage",
 597                         help="iso imaging tool")
 598     parser.add_argument("--config", "-c", default=None,
 599                         help="Provide config yaml for configuration. "\
 600                         "See config_example.yaml for example.")
 601     parser.add_argument("--efi-aarch64",
 602                         default="/usr/share/qemu-efi-aarch64/QEMU_EFI.fd",
 603                         help="Path to efi image for aarch64 VMs.")
 604     parser.add_argument("--log-console", action="store_true",
 605                         help="Log console to file.")
 606     parser.add_argument("commands", nargs="*", help="""Remaining
 607         commands after -- are passed to command inside the VM""")
 608
 609     return parser.parse_args()
 610
 611 def main(vmcls, config=None):
 612     try:
 613         if config == None:
 614             config = DEFAULT_CONFIG
 615         args = parse_args(vmcls)
 616         if not args.commands and not args.build_qemu and not args.build_image:
 617             print("Nothing to do?")
 618             return 1
 619         config = parse_config(config, args)
 620         logging.basicConfig(level=(logging.DEBUG if args.debug
 621                                    else logging.WARN))
 622         vm = vmcls(args, config=config)
 623         if args.build_image:
 624             if os.path.exists(args.image) and not args.force:
 625                 sys.stderr.writelines(["Image file exists: %s\n" % args.image,
 626                                       "Use --force option to overwrite\n"])
 627                 return 1
 628             return vm.build_image(args.image)
 629         if args.build_qemu:
 630             vm.add_source_dir(args.build_qemu)
 631             cmd = [vm.BUILD_SCRIPT.format(
 632                    configure_opts = " ".join(args.commands),
 633                    jobs=int(args.jobs),
 634                    target=args.build_target,
 635                    verbose = "V=1" if args.verbose else "")]
 636         else:
 637             cmd = args.commands
 638         img = args.image
 639         if args.snapshot:
 640             img += ",snapshot=on"
 641         vm.boot(img)
 642         vm.wait_ssh()
 643     except Exception as e:
 644         if isinstance(e, SystemExit) and e.code == 0:
 645             return 0
 646         sys.stderr.write("Failed to prepare guest environment\n")
 647         traceback.print_exc()
 648         return 2
 649
 650     exitcode = 0
 651     if vm.ssh(*cmd) != 0:
 652         exitcode = 3
 653     if args.interactive:
 654         vm.ssh()
 655
 656     if not args.snapshot:
 657         vm.graceful_shutdown()
 658
 659     return exitcode